Examples with ExpressionVirtualColumn org.apache.druid.segment.virtual.ExpressionVirtualColumn used on opensource projects

Example 16 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class GroupByQueryRunnerTest method testMultiValueDimensionAsArray.

@Test
public void testMultiValueDimensionAsArray() {
    // array types don't work with group by v1
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(UnsupportedOperationException.class);
        expectedException.expectMessage("GroupBy v1 only supports dimensions with an outputType of STRING.");
    }
    // Cannot vectorize due to multi-value dimensions.
    cannotVectorize();
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setVirtualColumns(new ExpressionVirtualColumn("v0", "mv_to_array(placementish)", ColumnType.STRING_ARRAY, ExprMacroTable.nil())).setDimensions(new DefaultDimensionSpec("v0", "alias", ColumnType.STRING_ARRAY)).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setGranularity(QueryRunnerTestHelper.ALL_GRAN).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("a", "preferred"), "rows", 2L, "idx", 282L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("b", "preferred"), "rows", 2L, "idx", 230L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("e", "preferred"), "rows", 2L, "idx", 324L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("h", "preferred"), "rows", 2L, "idx", 233L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("m", "preferred"), "rows", 6L, "idx", 5317L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("n", "preferred"), "rows", 2L, "idx", 235L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("p", "preferred"), "rows", 6L, "idx", 5405L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("preferred", "t"), "rows", 4L, "idx", 420L));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "multi-value-dim-groupby-arrays");
}

Example 17 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class GroupByQueryRunnerTest method testMultiValueDimensionAsArrayWithOtherDims.

@Test
public void testMultiValueDimensionAsArrayWithOtherDims() {
    // array types don't work with group by v1
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(UnsupportedOperationException.class);
        expectedException.expectMessage("GroupBy v1 only supports dimensions with an outputType of STRING");
    }
    // Cannot vectorize due to multi-value dimensions.
    cannotVectorize();
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setVirtualColumns(new ExpressionVirtualColumn("v0", "mv_to_array(placementish)", ColumnType.STRING_ARRAY, ExprMacroTable.nil())).setDimensions(new DefaultDimensionSpec("v0", "alias", ColumnType.STRING_ARRAY), new DefaultDimensionSpec("quality", "quality")).setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("alias", OrderByColumnSpec.Direction.ASCENDING, StringComparators.LEXICOGRAPHIC), new OrderByColumnSpec("quality", OrderByColumnSpec.Direction.ASCENDING, StringComparators.LEXICOGRAPHIC)), Integer.MAX_VALUE - 1)).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setGranularity(QueryRunnerTestHelper.ALL_GRAN).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("a", "preferred"), "quality", "automotive", "rows", 2L, "idx", 282L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("b", "preferred"), "quality", "business", "rows", 2L, "idx", 230L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("e", "preferred"), "quality", "entertainment", "rows", 2L, "idx", 324L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("h", "preferred"), "quality", "health", "rows", 2L, "idx", 233L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("m", "preferred"), "quality", "mezzanine", "rows", 6L, "idx", 5317L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("n", "preferred"), "quality", "news", "rows", 2L, "idx", 235L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("p", "preferred"), "quality", "premium", "rows", 6L, "idx", 5405L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("preferred", "t"), "quality", "technology", "rows", 2L, "idx", 175L), makeRow(query, "2011-04-01", "alias", ComparableStringArray.of("preferred", "t"), "quality", "travel", "rows", 2L, "idx", 245L));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "multi-value-dims-groupby-arrays");
    query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setVirtualColumns(new ExpressionVirtualColumn("v0", "mv_to_array(placementish)", ColumnType.STRING_ARRAY, ExprMacroTable.nil())).setDimensions(new DefaultDimensionSpec("v0", "alias", ColumnType.STRING_ARRAY), new DefaultDimensionSpec("quality", "quality")).setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("alias", OrderByColumnSpec.Direction.DESCENDING, StringComparators.LEXICOGRAPHIC), new OrderByColumnSpec("quality", OrderByColumnSpec.Direction.DESCENDING, StringComparators.LEXICOGRAPHIC)), Integer.MAX_VALUE - 1)).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setGranularity(QueryRunnerTestHelper.ALL_GRAN).build();
    Collections.reverse(expectedResults);
    results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "multi-value-dims-groupby-arrays-descending");
}

Example 18 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionFilterOnRHSRegionNameExprVirtualColumn.

@Test
public void test_filterPushDown_factToRegionFilterOnRHSRegionNameExprVirtualColumn() {
    // If our query had a filter that uses expressions, such as upper(r1.regionName) == 'VIRGINIA', this plans into
    // a selector filter on an ExpressionVirtualColumn
    Filter originalFilter = new SelectorFilter("v0", "VIRGINIA");
    VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v0", "upper(\"r1.regionName\")", ColumnType.STRING, TestExprMacroTable.INSTANCE)));
    JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT)));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses.getJoinableClauses(), virtualColumns);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", "v0"), ImmutableList.of(new Object[] { "Old Anatolian Turkish", "VIRGINIA" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(null, new SelectorFilter("v0", "VIRGINIA"), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 19 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper.

private void test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper(boolean hasLhsExpressionInJoinCondition) {
    Filter expressionFilter = new ExpressionDimFilter("\"rtc.countryIsoCode\" == 'CA'", ExprMacroTable.nil()).toFilter();
    Filter specialSelectorFilter = new SelectorFilter("rtc.countryIsoCode", "CA") {

        @Override
        public boolean supportsRequiredColumnRewrite() {
            return false;
        }
    };
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("r1.regionIsoCode", "ON"), new SelectorFilter("rtc.countryIsoCode", "CA"), specialSelectorFilter, new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "CA", "CB", false, false, null, null, null)), expressionFilter, new InDimFilter("rtc.countryIsoCode", ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter("rtc.countryIsoCode", "QQQ"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null))))));
    JoinableClause factToRegionClause;
    if (hasLhsExpressionInJoinCondition) {
        factToRegionClause = new JoinableClause(FACT_TO_REGION_PREFIX, new IndexedTableJoinable(regionsTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sregionIsoCode\" == upper(lower(regionIsoCode)) && \"%scountryIsoCode\" == upper(lower(countryIsoCode))", FACT_TO_REGION_PREFIX, FACT_TO_REGION_PREFIX), FACT_TO_REGION_PREFIX, ExprMacroTable.nil()));
    } else {
        factToRegionClause = factToRegion(JoinType.LEFT);
    }
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegionClause, regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    String rewrittenCountryIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-1" : "countryIsoCode";
    String rewrittenRegionIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0" : "regionIsoCode";
    Set<VirtualColumn> expectedVirtualColumns;
    if (hasLhsExpressionInJoinCondition) {
        expectedVirtualColumns = ImmutableSet.of(new ExpressionVirtualColumn(rewrittenRegionIsoCodeColumnName, "(upper [(lower [regionIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()), new ExpressionVirtualColumn(rewrittenCountryIsoCodeColumnName, "(upper [(lower [countryIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()));
    } else {
        expectedVirtualColumns = ImmutableSet.of();
    }
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Didier Leclair", "Ontario", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter(rewrittenRegionIsoCodeColumnName, "ON"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "CA"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "CA", "CB", false, false, null, null, null)), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "QQQ"), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "ABCDEF"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), new AndFilter(ImmutableList.of(specialSelectorFilter, expressionFilter, new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), expectedVirtualColumns);
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 20 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnChannelVirtualColumn.

@Test
public void test_filterPushDown_factToRegionToCountryLeftFilterOnChannelVirtualColumn() {
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT));
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("v1", "virtual-column-#en.wikipedia")));
    VirtualColumns virtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("v1", "concat('virtual-column-', \"channel\")", ColumnType.STRING, TestExprMacroTable.INSTANCE)));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, virtualColumns);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, virtualColumns, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Talk:Oswald Tilghman", null, null }, new Object[] { "Peremptory norm", "New South Wales", "Australia" }, new Object[] { "President of India", "California", "United States" }, new Object[] { "Glasgow", "Kingston upon Hull", "United Kingdom" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "Sarah Michelle Gellar", "Ontario", "Canada" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Giusy Ferreri discography", "Provincia di Varese", "Italy" }, new Object[] { "Roma-Bangkok", "Provincia di Varese", "Italy" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }, new Object[] { "Cream Soda", "Ainigriv", "States United" }, new Object[] { "Orange Soda", null, null }, new Object[] { "History of Fourems", "Fourems Province", "Fourems" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new SelectorFilter("v1", "virtual-column-#en.wikipedia"), null, ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}