Examples with OrFilter org.apache.druid.segment.filter.OrFilter used on opensource projects

Example 1 with OrFilter

use of org.apache.druid.segment.filter.OrFilter in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteOrFilter.

/**
 * Potentially rewrite the subfilters of an OR filter so that the whole OR filter can be pushed down to
 * the base table.
 *
 * @param orFilter                          OrFilter to be rewritten
 * @param joinFilterPreAnalysis             The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param pushDownVirtualColumnsForLhsExprs See comments on {@link #analyzeJoinFilterClause}
 *
 * @return A JoinFilterAnalysis indicating how to handle the potentially rewritten filter
 */
private static JoinFilterAnalysis rewriteOrFilter(OrFilter orFilter, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    List<Filter> newFilters = new ArrayList<>();
    boolean retainRhs = false;
    for (Filter filter : orFilter.getFilters()) {
        if (!joinFilterPreAnalysis.getJoinableClauses().areSomeColumnsFromJoin(filter.getRequiredColumns())) {
            newFilters.add(filter);
            continue;
        }
        JoinFilterAnalysis rewritten = null;
        if (joinFilterPreAnalysis.getEquiconditions().doesFilterSupportDirectJoinFilterRewrite(filter)) {
            rewritten = rewriteFilterDirect(filter, joinFilterPreAnalysis, pushDownVirtualColumnsForLhsExprs);
        } else if (filter instanceof SelectorFilter) {
            retainRhs = true;
            // We could optimize retainRhs handling further by introducing a "filter to retain" property to the
            // analysis, and only keeping the subfilters that need to be retained
            rewritten = rewriteSelectorFilter((SelectorFilter) filter, joinFilterPreAnalysis, pushDownVirtualColumnsForLhsExprs);
        }
        if (rewritten == null || !rewritten.isCanPushDown()) {
            return JoinFilterAnalysis.createNoPushdownFilterAnalysis(orFilter);
        } else {
            // noinspection OptionalGetWithoutIsPresent isCanPushDown checks isPresent
            newFilters.add(rewritten.getPushDownFilter().get());
        }
    }
    return new JoinFilterAnalysis(retainRhs, orFilter, Filters.maybeOr(newFilters).orElse(null));
}

Example 2 with OrFilter

use of org.apache.druid.segment.filter.OrFilter in project druid by druid-io.

the class RhsRewriteCandidates method getRhsRewriteCandidates.

/**
 * Determine candidates for filter rewrites.
 * A candidate is an RHS column that appears in a filter, along with the value being filtered on, plus
 * the joinable clause associated with the table that the RHS column is from.
 *
 * These candidates are redued to filter rewrite correlations.
 *
 * @param normalizedJoinTableClauses
 * @param equiconditions
 * @param joinableClauses
 * @return A set of candidates for filter rewrites.
 */
public static RhsRewriteCandidates getRhsRewriteCandidates(List<Filter> normalizedJoinTableClauses, Equiconditions equiconditions, JoinableClauses joinableClauses) {
    Set<RhsRewriteCandidate> rhsRewriteCandidates = new LinkedHashSet<>();
    for (Filter orClause : normalizedJoinTableClauses) {
        if (Filters.filterMatchesNull(orClause)) {
            continue;
        }
        if (orClause instanceof OrFilter) {
            for (Filter subFilter : ((OrFilter) orClause).getFilters()) {
                Optional<RhsRewriteCandidate> rhsRewriteCandidate = determineRhsRewriteCandidatesForSingleFilter(subFilter, equiconditions, joinableClauses);
                rhsRewriteCandidate.ifPresent(rhsRewriteCandidates::add);
            }
            continue;
        }
        Optional<RhsRewriteCandidate> rhsRewriteCandidate = determineRhsRewriteCandidatesForSingleFilter(orClause, equiconditions, joinableClauses);
        rhsRewriteCandidate.ifPresent(rhsRewriteCandidates::add);
    }
    return new RhsRewriteCandidates(rhsRewriteCandidates);
}

Example 3 with OrFilter

use of org.apache.druid.segment.filter.OrFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite.

@Test
public void test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite() {
    JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT)));
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null))));
    JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(true, false, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses.getJoinableClauses(), VirtualColumns.EMPTY, originalFilter));
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "President of India", "California", "United States" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null)))), new AndFilter(ImmutableList.of(new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))))), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 4 with OrFilter

use of org.apache.druid.segment.filter.OrFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper.

private void test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper(boolean hasLhsExpressionInJoinCondition) {
    Filter expressionFilter = new ExpressionDimFilter("\"rtc.countryIsoCode\" == 'CA'", ExprMacroTable.nil()).toFilter();
    Filter specialSelectorFilter = new SelectorFilter("rtc.countryIsoCode", "CA") {

        @Override
        public boolean supportsRequiredColumnRewrite() {
            return false;
        }
    };
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("r1.regionIsoCode", "ON"), new SelectorFilter("rtc.countryIsoCode", "CA"), specialSelectorFilter, new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "CA", "CB", false, false, null, null, null)), expressionFilter, new InDimFilter("rtc.countryIsoCode", ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter("rtc.countryIsoCode", "QQQ"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null))))));
    JoinableClause factToRegionClause;
    if (hasLhsExpressionInJoinCondition) {
        factToRegionClause = new JoinableClause(FACT_TO_REGION_PREFIX, new IndexedTableJoinable(regionsTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sregionIsoCode\" == upper(lower(regionIsoCode)) && \"%scountryIsoCode\" == upper(lower(countryIsoCode))", FACT_TO_REGION_PREFIX, FACT_TO_REGION_PREFIX), FACT_TO_REGION_PREFIX, ExprMacroTable.nil()));
    } else {
        factToRegionClause = factToRegion(JoinType.LEFT);
    }
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegionClause, regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    String rewrittenCountryIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-1" : "countryIsoCode";
    String rewrittenRegionIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0" : "regionIsoCode";
    Set<VirtualColumn> expectedVirtualColumns;
    if (hasLhsExpressionInJoinCondition) {
        expectedVirtualColumns = ImmutableSet.of(new ExpressionVirtualColumn(rewrittenRegionIsoCodeColumnName, "(upper [(lower [regionIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()), new ExpressionVirtualColumn(rewrittenCountryIsoCodeColumnName, "(upper [(lower [countryIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()));
    } else {
        expectedVirtualColumns = ImmutableSet.of();
    }
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Didier Leclair", "Ontario", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter(rewrittenRegionIsoCodeColumnName, "ON"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "CA"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "CA", "CB", false, false, null, null, null)), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "QQQ"), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "ABCDEF"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), new AndFilter(ImmutableList.of(specialSelectorFilter, expressionFilter, new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), expectedVirtualColumns);
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 5 with OrFilter

use of org.apache.druid.segment.filter.OrFilter in project druid by druid-io.

the class FilterPartitionBenchmark method readOrFilterCNF.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readOrFilterCNF(Blackhole blackhole) {
    Filter filter = new NoBitmapSelectorFilter("dimSequential", "199");
    Filter filter2 = new AndFilter(Arrays.asList(new SelectorFilter("dimMultivalEnumerated2", "Corundum"), new NoBitmapSelectorFilter("dimMultivalEnumerated", "Bar")));
    Filter orFilter = new OrFilter(Arrays.asList(filter, filter2));
    StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
    Sequence<Cursor> cursors = makeCursors(sa, Filters.toCnf(orFilter));
    readCursors(cursors, blackhole);
}