Examples with BoundDimFilter org.apache.druid.query.filter.BoundDimFilter used on opensource projects

Example 21 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class NativeQueryMaker method runQuery.

@Override
public Sequence<Object[]> runQuery(final DruidQuery druidQuery) {
    final Query<?> query = druidQuery.getQuery();
    if (plannerContext.getPlannerConfig().isRequireTimeCondition() && !(druidQuery.getDataSource() instanceof InlineDataSource)) {
        if (Intervals.ONLY_ETERNITY.equals(findBaseDataSourceIntervals(query))) {
            throw new CannotBuildQueryException("requireTimeCondition is enabled, all queries must include a filter condition on the __time column");
        }
    }
    int numFilters = plannerContext.getPlannerConfig().getMaxNumericInFilters();
    // Instead of IN(v1,v2,v3) user should specify IN('v1','v2','v3')
    if (numFilters != PlannerConfig.NUM_FILTER_NOT_USED) {
        if (query.getFilter() instanceof OrDimFilter) {
            OrDimFilter orDimFilter = (OrDimFilter) query.getFilter();
            int numBoundFilters = 0;
            for (DimFilter filter : orDimFilter.getFields()) {
                numBoundFilters += filter instanceof BoundDimFilter ? 1 : 0;
            }
            if (numBoundFilters > numFilters) {
                String dimension = ((BoundDimFilter) (orDimFilter.getFields().get(0))).getDimension();
                throw new UOE(StringUtils.format("The number of values in the IN clause for [%s] in query exceeds configured maxNumericFilter limit of [%s] for INs. Cast [%s] values of IN clause to String", dimension, numFilters, orDimFilter.getFields().size()));
            }
        }
    }
    final List<String> rowOrder;
    if (query instanceof TimeseriesQuery && !druidQuery.getGrouping().getDimensions().isEmpty()) {
        // Hack for timeseries queries: when generating them, DruidQuery.toTimeseriesQuery translates a dimension
        // based on a timestamp_floor expression into a 'granularity'. This is not reflected in the druidQuery's
        // output row signature, so we have to account for it here.
        // TODO: We can remove this once https://github.com/apache/druid/issues/9974 is done.
        final String timeDimension = Iterables.getOnlyElement(druidQuery.getGrouping().getDimensions()).getOutputName();
        rowOrder = druidQuery.getOutputRowSignature().getColumnNames().stream().map(f -> timeDimension.equals(f) ? ColumnHolder.TIME_COLUMN_NAME : f).collect(Collectors.toList());
    } else {
        rowOrder = druidQuery.getOutputRowSignature().getColumnNames();
    }
    final List<SqlTypeName> columnTypes = druidQuery.getOutputRowType().getFieldList().stream().map(f -> f.getType().getSqlTypeName()).collect(Collectors.toList());
    return execute(query, mapColumnList(rowOrder, fieldMapping), mapColumnList(columnTypes, fieldMapping));
}

Example 22 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite.

@Test
public void test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite() {
    JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT)));
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null))));
    JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(true, false, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses.getJoinableClauses(), VirtualColumns.EMPTY, originalFilter));
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "President of India", "California", "United States" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null)))), new AndFilter(ImmutableList.of(new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))))), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 23 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper.

private void test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper(boolean hasLhsExpressionInJoinCondition) {
    Filter expressionFilter = new ExpressionDimFilter("\"rtc.countryIsoCode\" == 'CA'", ExprMacroTable.nil()).toFilter();
    Filter specialSelectorFilter = new SelectorFilter("rtc.countryIsoCode", "CA") {

        @Override
        public boolean supportsRequiredColumnRewrite() {
            return false;
        }
    };
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("r1.regionIsoCode", "ON"), new SelectorFilter("rtc.countryIsoCode", "CA"), specialSelectorFilter, new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "CA", "CB", false, false, null, null, null)), expressionFilter, new InDimFilter("rtc.countryIsoCode", ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter("rtc.countryIsoCode", "QQQ"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null))))));
    JoinableClause factToRegionClause;
    if (hasLhsExpressionInJoinCondition) {
        factToRegionClause = new JoinableClause(FACT_TO_REGION_PREFIX, new IndexedTableJoinable(regionsTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sregionIsoCode\" == upper(lower(regionIsoCode)) && \"%scountryIsoCode\" == upper(lower(countryIsoCode))", FACT_TO_REGION_PREFIX, FACT_TO_REGION_PREFIX), FACT_TO_REGION_PREFIX, ExprMacroTable.nil()));
    } else {
        factToRegionClause = factToRegion(JoinType.LEFT);
    }
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegionClause, regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    String rewrittenCountryIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-1" : "countryIsoCode";
    String rewrittenRegionIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0" : "regionIsoCode";
    Set<VirtualColumn> expectedVirtualColumns;
    if (hasLhsExpressionInJoinCondition) {
        expectedVirtualColumns = ImmutableSet.of(new ExpressionVirtualColumn(rewrittenRegionIsoCodeColumnName, "(upper [(lower [regionIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()), new ExpressionVirtualColumn(rewrittenCountryIsoCodeColumnName, "(upper [(lower [countryIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()));
    } else {
        expectedVirtualColumns = ImmutableSet.of();
    }
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Didier Leclair", "Ontario", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter(rewrittenRegionIsoCodeColumnName, "ON"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "CA"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "CA", "CB", false, false, null, null, null)), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "QQQ"), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "ABCDEF"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), new AndFilter(ImmutableList.of(specialSelectorFilter, expressionFilter, new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), expectedVirtualColumns);
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 24 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class DruidQueryTest method test_filtration_intervalsInBothFilters.

@Test
public void test_filtration_intervalsInBothFilters() {
    DataSource dataSource = join(JoinType.INNER, filterWithInterval);
    DataSource expectedDataSource = join(JoinType.INNER, selectorFilter);
    DimFilter queryFilter = new AndDimFilter(otherFilter, new BoundDimFilter("__time", "150", "250", false, true, null, null, StringComparators.NUMERIC));
    Pair<DataSource, Filtration> pair = DruidQuery.getFiltration(dataSource, queryFilter, VirtualColumnRegistry.create(RowSignature.empty(), TestExprMacroTable.INSTANCE));
    verify(pair, expectedDataSource, otherFilter, Intervals.utc(150, 200));
}

Example 25 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class FilteredAggregatorTest method testAggregateWithPredicateFilters.

@Test
public void testAggregateWithPredicateFilters() {
    final float[] values = { 0.15f, 0.27f };
    TestFloatColumnSelector selector;
    FilteredAggregatorFactory factory;
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "a", "a", false, false, true, null, StringComparators.ALPHANUMERIC));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new RegexDimFilter("dim", "a", null));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a", true), null));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    String jsFn = "function(x) { return(x === 'a') }";
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new JavaScriptDimFilter("dim", jsFn, null, JavaScriptConfig.getEnabledInstance()));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
}