Examples with ExpressionDimFilter org.apache.druid.query.filter.ExpressionDimFilter used on opensource projects

Example 1 with ExpressionDimFilter

use of org.apache.druid.query.filter.ExpressionDimFilter in project druid by druid-io.

the class CalciteArraysQueryTest method testArrayAggGroupByArrayContainsSubquery.

@Test
public void testArrayAggGroupByArrayContainsSubquery() throws Exception {
    cannotVectorize();
    List<Object[]> expectedResults;
    if (useDefault) {
        expectedResults = ImmutableList.of(new Object[] { "", 3L }, new Object[] { "a", 1L }, new Object[] { "abc", 1L });
    } else {
        expectedResults = ImmutableList.of(new Object[] { null, 2L }, new Object[] { "", 1L }, new Object[] { "a", 2L }, new Object[] { "abc", 1L });
    }
    testQuery("SELECT dim2, COUNT(*) FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1) GROUP BY 1", ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.DATASOURCE1), new QueryDataSource(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).filters(not(selector("dim1", null, null))).aggregators(aggregators(new ExpressionLambdaAggregatorFactory("a0", ImmutableSet.of("dim1"), "__acc", "ARRAY<STRING>[]", "ARRAY<STRING>[]", true, true, false, "array_set_add(\"__acc\", \"dim1\")", "array_set_add_all(\"__acc\", \"a0\")", null, null, ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE))).context(QUERY_CONTEXT_DEFAULT).build()), "j0.", "1", JoinType.LEFT, null)).setInterval(querySegmentSpec(Filtration.eternity())).setDimFilter(new ExpressionDimFilter("array_contains(\"j0.a0\",\"dim1\")", TestExprMacroTable.INSTANCE)).setDimensions(dimensions(new DefaultDimensionSpec("dim2", "d0"))).setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))).setGranularity(Granularities.ALL).setLimitSpec(NoopLimitSpec.instance()).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 2 with ExpressionDimFilter

use of org.apache.druid.query.filter.ExpressionDimFilter in project druid by druid-io.

the class CalciteArraysQueryTest method testArrayAggArrayContainsSubquery.

@Test
public void testArrayAggArrayContainsSubquery() throws Exception {
    cannotVectorize();
    List<Object[]> expectedResults;
    if (useDefault) {
        expectedResults = ImmutableList.of(new Object[] { "10.1", "" }, new Object[] { "2", "" }, new Object[] { "1", "a" }, new Object[] { "def", "abc" }, new Object[] { "abc", "" });
    } else {
        expectedResults = ImmutableList.of(new Object[] { "", "a" }, new Object[] { "10.1", null }, new Object[] { "2", "" }, new Object[] { "1", "a" }, new Object[] { "def", "abc" }, new Object[] { "abc", null });
    }
    testQuery("SELECT dim1,dim2 FROM foo WHERE ARRAY_CONTAINS((SELECT ARRAY_AGG(DISTINCT dim1) FROM foo WHERE dim1 is not null), dim1)", ImmutableList.of(Druids.newScanQueryBuilder().dataSource(join(new TableDataSource(CalciteTests.DATASOURCE1), new QueryDataSource(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).filters(not(selector("dim1", null, null))).aggregators(aggregators(new ExpressionLambdaAggregatorFactory("a0", ImmutableSet.of("dim1"), "__acc", "ARRAY<STRING>[]", "ARRAY<STRING>[]", true, true, false, "array_set_add(\"__acc\", \"dim1\")", "array_set_add_all(\"__acc\", \"a0\")", null, null, ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE))).context(QUERY_CONTEXT_DEFAULT).build()), "j0.", "1", JoinType.LEFT, null)).intervals(querySegmentSpec(Filtration.eternity())).filters(new ExpressionDimFilter("array_contains(\"j0.a0\",\"dim1\")", TestExprMacroTable.INSTANCE)).columns("dim1", "dim2").context(QUERY_CONTEXT_DEFAULT).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST).legacy(false).build()), expectedResults);
}

Example 3 with ExpressionDimFilter

use of org.apache.druid.query.filter.ExpressionDimFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper.

private void test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper(boolean hasLhsExpressionInJoinCondition) {
    Filter expressionFilter = new ExpressionDimFilter("\"rtc.countryIsoCode\" == 'CA'", ExprMacroTable.nil()).toFilter();
    Filter specialSelectorFilter = new SelectorFilter("rtc.countryIsoCode", "CA") {

        @Override
        public boolean supportsRequiredColumnRewrite() {
            return false;
        }
    };
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("r1.regionIsoCode", "ON"), new SelectorFilter("rtc.countryIsoCode", "CA"), specialSelectorFilter, new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "CA", "CB", false, false, null, null, null)), expressionFilter, new InDimFilter("rtc.countryIsoCode", ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter("rtc.countryIsoCode", "QQQ"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null))))));
    JoinableClause factToRegionClause;
    if (hasLhsExpressionInJoinCondition) {
        factToRegionClause = new JoinableClause(FACT_TO_REGION_PREFIX, new IndexedTableJoinable(regionsTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sregionIsoCode\" == upper(lower(regionIsoCode)) && \"%scountryIsoCode\" == upper(lower(countryIsoCode))", FACT_TO_REGION_PREFIX, FACT_TO_REGION_PREFIX), FACT_TO_REGION_PREFIX, ExprMacroTable.nil()));
    } else {
        factToRegionClause = factToRegion(JoinType.LEFT);
    }
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegionClause, regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    String rewrittenCountryIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-1" : "countryIsoCode";
    String rewrittenRegionIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0" : "regionIsoCode";
    Set<VirtualColumn> expectedVirtualColumns;
    if (hasLhsExpressionInJoinCondition) {
        expectedVirtualColumns = ImmutableSet.of(new ExpressionVirtualColumn(rewrittenRegionIsoCodeColumnName, "(upper [(lower [regionIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()), new ExpressionVirtualColumn(rewrittenCountryIsoCodeColumnName, "(upper [(lower [countryIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()));
    } else {
        expectedVirtualColumns = ImmutableSet.of();
    }
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Didier Leclair", "Ontario", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter(rewrittenRegionIsoCodeColumnName, "ON"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "CA"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "CA", "CB", false, false, null, null, null)), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "QQQ"), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "ABCDEF"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), new AndFilter(ImmutableList.of(specialSelectorFilter, expressionFilter, new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), expectedVirtualColumns);
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 4 with ExpressionDimFilter

use of org.apache.druid.query.filter.ExpressionDimFilter in project druid by druid-io.

the class HashJoinSegmentStorageAdapterTest method test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinConditionUsingLookup.

@Test
public void test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinConditionUsingLookup() {
    // Join condition => always true.
    // Filter => Fact to countries on countryIsoCode.
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause(FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, LookupJoinable.wrap(countryIsoCodeToNameLookup), JoinType.INNER, JoinConditionAnalysis.forExpression("1", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, ExprMacroTable.nil())));
    Filter filter = new ExpressionDimFilter(StringUtils.format("\"%sk\" == countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX), ExprMacroTable.nil()).toFilter();
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(filter, joinableClauses, VirtualColumns.EMPTY);
    JoinTestHelper.verifyCursors(new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis).makeCursors(filter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", "countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "k", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "v"), ImmutableList.of(new Object[] { "Peremptory norm", "AU", "AU", "Australia" }, new Object[] { "Mathis Bolly", "MX", "MX", "Mexico" }, new Object[] { "유희왕 GX", "KR", "KR", "Republic of Korea" }, new Object[] { "青野武", "JP", "JP", "Japan" }, new Object[] { "Golpe de Estado en Chile de 1973", "CL", "CL", "Chile" }, new Object[] { "President of India", "US", "US", "United States" }, new Object[] { "Diskussion:Sebastian Schulz", "DE", "DE", "Germany" }, new Object[] { "Saison 9 de Secret Story", "FR", "FR", "France" }, new Object[] { "Glasgow", "GB", "GB", "United Kingdom" }, new Object[] { "Didier Leclair", "CA", "CA", "Canada" }, new Object[] { "Les Argonautes", "CA", "CA", "Canada" }, new Object[] { "Otjiwarongo Airport", "US", "US", "United States" }, new Object[] { "Sarah Michelle Gellar", "CA", "CA", "Canada" }, new Object[] { "DirecTV", "US", "US", "United States" }, new Object[] { "Carlo Curti", "US", "US", "United States" }, new Object[] { "Giusy Ferreri discography", "IT", "IT", "Italy" }, new Object[] { "Roma-Bangkok", "IT", "IT", "Italy" }, new Object[] { "Wendigo", "SV", "SV", "El Salvador" }, new Object[] { "Алиса в Зазеркалье", "NO", "NO", "Norway" }, new Object[] { "Gabinete Ministerial de Rafael Correa", "EC", "EC", "Ecuador" }, new Object[] { "Old Anatolian Turkish", "US", "US", "United States" }, new Object[] { "Cream Soda", "SU", "SU", "States United" }, new Object[] { "History of Fourems", "MMMM", "MMMM", "Fourems" }));
}

Example 5 with ExpressionDimFilter

use of org.apache.druid.query.filter.ExpressionDimFilter in project druid by druid-io.

the class HashJoinSegmentStorageAdapterTest method test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinCondition.

@Test
public void test_makeCursors_factToCountryInnerWithFilterInsteadOfRealJoinCondition() {
    // Join condition => always true.
    // Filter => Fact to countries on countryIsoCode.
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause(FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, new IndexedTableJoinable(countriesTable), JoinType.INNER, JoinConditionAnalysis.forExpression("1", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, ExprMacroTable.nil())));
    Filter filter = new ExpressionDimFilter(StringUtils.format("\"%scountryIsoCode\" == countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX), ExprMacroTable.nil()).toFilter();
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(filter, joinableClauses, VirtualColumns.EMPTY);
    JoinTestHelper.verifyCursors(new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis).makeCursors(filter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", "countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryIsoCode", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryNumber"), ImmutableList.of(new Object[] { "Peremptory norm", "AU", "AU", "Australia", 0L }, new Object[] { "Mathis Bolly", "MX", "MX", "Mexico", 10L }, new Object[] { "유희왕 GX", "KR", "KR", "Republic of Korea", 9L }, new Object[] { "青野武", "JP", "JP", "Japan", 8L }, new Object[] { "Golpe de Estado en Chile de 1973", "CL", "CL", "Chile", 2L }, new Object[] { "President of India", "US", "US", "United States", 13L }, new Object[] { "Diskussion:Sebastian Schulz", "DE", "DE", "Germany", 3L }, new Object[] { "Saison 9 de Secret Story", "FR", "FR", "France", 5L }, new Object[] { "Glasgow", "GB", "GB", "United Kingdom", 6L }, new Object[] { "Didier Leclair", "CA", "CA", "Canada", 1L }, new Object[] { "Les Argonautes", "CA", "CA", "Canada", 1L }, new Object[] { "Otjiwarongo Airport", "US", "US", "United States", 13L }, new Object[] { "Sarah Michelle Gellar", "CA", "CA", "Canada", 1L }, new Object[] { "DirecTV", "US", "US", "United States", 13L }, new Object[] { "Carlo Curti", "US", "US", "United States", 13L }, new Object[] { "Giusy Ferreri discography", "IT", "IT", "Italy", 7L }, new Object[] { "Roma-Bangkok", "IT", "IT", "Italy", 7L }, new Object[] { "Wendigo", "SV", "SV", "El Salvador", 12L }, new Object[] { "Алиса в Зазеркалье", "NO", "NO", "Norway", 11L }, new Object[] { "Gabinete Ministerial de Rafael Correa", "EC", "EC", "Ecuador", 4L }, new Object[] { "Old Anatolian Turkish", "US", "US", "United States", 13L }, new Object[] { "Cream Soda", "SU", "SU", "States United", 15L }, new Object[] { "History of Fourems", "MMMM", "MMMM", "Fourems", 205L }));
}