Examples with Filter org.apache.druid.query.filter.Filter used on opensource projects

Example 16 with Filter

use of org.apache.druid.query.filter.Filter in project druid by druid-io.

the class JoinableFactoryWrapper method convertJoinsToFilters.

/**
 * Converts any join clauses to filters that can be converted, and returns the rest as-is.
 *
 * See {@link #convertJoinToFilter} for details on the logic.
 */
@VisibleForTesting
static Pair<List<Filter>, List<JoinableClause>> convertJoinsToFilters(final List<JoinableClause> clauses, final Set<String> requiredColumns, final int maxNumFilterValues) {
    final List<Filter> filterList = new ArrayList<>();
    final List<JoinableClause> clausesToUse = new ArrayList<>();
    // Join clauses may depend on other, earlier join clauses.
    // We track that using a Multiset, because we'll need to remove required columns one by one as we convert clauses,
    // and multiple clauses may refer to the same column.
    final Multiset<String> columnsRequiredByJoinClauses = HashMultiset.create();
    for (JoinableClause clause : clauses) {
        for (String column : clause.getCondition().getRequiredColumns()) {
            columnsRequiredByJoinClauses.add(column, 1);
        }
    }
    // Walk through the list of clauses, picking off any from the start of the list that can be converted to filters.
    boolean atStart = true;
    for (JoinableClause clause : clauses) {
        if (atStart) {
            // Remove this clause from columnsRequiredByJoinClauses. It's ok if it relies on itself.
            for (String column : clause.getCondition().getRequiredColumns()) {
                columnsRequiredByJoinClauses.remove(column, 1);
            }
            final Optional<Filter> filter = convertJoinToFilter(clause, Sets.union(requiredColumns, columnsRequiredByJoinClauses.elementSet()), maxNumFilterValues);
            if (filter.isPresent()) {
                filterList.add(filter.get());
            } else {
                clausesToUse.add(clause);
                atStart = false;
            }
        } else {
            clausesToUse.add(clause);
        }
    }
    // Sanity check. If this exception is ever thrown, it's a bug.
    if (filterList.size() + clausesToUse.size() != clauses.size()) {
        throw new ISE("Lost a join clause during planning");
    }
    return Pair.of(filterList, clausesToUse);
}

Example 17 with Filter

use of org.apache.druid.query.filter.Filter in project druid by druid-io.

the class JoinFilterAnalyzer method splitFilter.

/**
 * @param joinFilterPreAnalysis The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param baseFilter - Filter on base table that was specified in the query itself
 *
 * @return A JoinFilterSplit indicating what parts of the filter should be applied pre-join and post-join
 */
public static JoinFilterSplit splitFilter(JoinFilterPreAnalysis joinFilterPreAnalysis, @Nullable Filter baseFilter) {
    if (joinFilterPreAnalysis.getOriginalFilter() == null || !joinFilterPreAnalysis.isEnableFilterPushDown()) {
        return new JoinFilterSplit(baseFilter, joinFilterPreAnalysis.getOriginalFilter(), ImmutableSet.of());
    }
    // Pushdown filters, rewriting if necessary
    List<Filter> leftFilters = new ArrayList<>();
    List<Filter> rightFilters = new ArrayList<>();
    Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs = new HashMap<>();
    if (null != baseFilter) {
        leftFilters.add(baseFilter);
    }
    for (Filter baseTableFilter : joinFilterPreAnalysis.getNormalizedBaseTableClauses()) {
        if (!Filters.filterMatchesNull(baseTableFilter)) {
            leftFilters.add(baseTableFilter);
        } else {
            rightFilters.add(baseTableFilter);
        }
    }
    for (Filter orClause : joinFilterPreAnalysis.getNormalizedJoinTableClauses()) {
        JoinFilterAnalysis joinFilterAnalysis = analyzeJoinFilterClause(orClause, joinFilterPreAnalysis, pushDownVirtualColumnsForLhsExprs);
        if (joinFilterAnalysis.isCanPushDown()) {
            // noinspection OptionalGetWithoutIsPresent isCanPushDown checks isPresent
            leftFilters.add(joinFilterAnalysis.getPushDownFilter().get());
        }
        if (joinFilterAnalysis.isRetainAfterJoin()) {
            rightFilters.add(joinFilterAnalysis.getOriginalFilter());
        }
    }
    return new JoinFilterSplit(Filters.maybeAnd(leftFilters).orElse(null), Filters.maybeAnd(rightFilters).orElse(null), new HashSet<>(pushDownVirtualColumnsForLhsExprs.values()));
}

Example 18 with Filter

use of org.apache.druid.query.filter.Filter in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteFilterDirect.

private static JoinFilterAnalysis rewriteFilterDirect(Filter filterClause, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    if (!filterClause.supportsRequiredColumnRewrite()) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(filterClause);
    }
    List<Filter> newFilters = new ArrayList<>();
    // we only support direct rewrites of filters that reference a single column
    String reqColumn = filterClause.getRequiredColumns().iterator().next();
    List<JoinFilterColumnCorrelationAnalysis> correlationAnalyses = joinFilterPreAnalysis.getCorrelationsByDirectFilteringColumn().get(reqColumn);
    if (correlationAnalyses == null) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(filterClause);
    }
    for (JoinFilterColumnCorrelationAnalysis correlationAnalysis : correlationAnalyses) {
        if (correlationAnalysis.supportsPushDown()) {
            for (String correlatedBaseColumn : correlationAnalysis.getBaseColumns()) {
                Filter rewrittenFilter = filterClause.rewriteRequiredColumns(ImmutableMap.of(reqColumn, correlatedBaseColumn));
                newFilters.add(rewrittenFilter);
            }
            for (Expr correlatedBaseExpr : correlationAnalysis.getBaseExpressions()) {
                // We need to create a virtual column for the expressions when pushing down
                VirtualColumn pushDownVirtualColumn = pushDownVirtualColumnsForLhsExprs.computeIfAbsent(correlatedBaseExpr, (expr) -> {
                    String vcName = getCorrelatedBaseExprVirtualColumnName(pushDownVirtualColumnsForLhsExprs.size());
                    return new ExpressionVirtualColumn(vcName, correlatedBaseExpr, ColumnType.STRING);
                });
                Filter rewrittenFilter = filterClause.rewriteRequiredColumns(ImmutableMap.of(reqColumn, pushDownVirtualColumn.getOutputName()));
                newFilters.add(rewrittenFilter);
            }
        }
    }
    if (newFilters.isEmpty()) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(filterClause);
    }
    return new JoinFilterAnalysis(false, filterClause, Filters.maybeAnd(newFilters).orElse(null));
}

Example 19 with Filter

use of org.apache.druid.query.filter.Filter in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteOrFilter.

/**
 * Potentially rewrite the subfilters of an OR filter so that the whole OR filter can be pushed down to
 * the base table.
 *
 * @param orFilter                          OrFilter to be rewritten
 * @param joinFilterPreAnalysis             The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param pushDownVirtualColumnsForLhsExprs See comments on {@link #analyzeJoinFilterClause}
 *
 * @return A JoinFilterAnalysis indicating how to handle the potentially rewritten filter
 */
private static JoinFilterAnalysis rewriteOrFilter(OrFilter orFilter, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    List<Filter> newFilters = new ArrayList<>();
    boolean retainRhs = false;
    for (Filter filter : orFilter.getFilters()) {
        if (!joinFilterPreAnalysis.getJoinableClauses().areSomeColumnsFromJoin(filter.getRequiredColumns())) {
            newFilters.add(filter);
            continue;
        }
        JoinFilterAnalysis rewritten = null;
        if (joinFilterPreAnalysis.getEquiconditions().doesFilterSupportDirectJoinFilterRewrite(filter)) {
            rewritten = rewriteFilterDirect(filter, joinFilterPreAnalysis, pushDownVirtualColumnsForLhsExprs);
        } else if (filter instanceof SelectorFilter) {
            retainRhs = true;
            // We could optimize retainRhs handling further by introducing a "filter to retain" property to the
            // analysis, and only keeping the subfilters that need to be retained
            rewritten = rewriteSelectorFilter((SelectorFilter) filter, joinFilterPreAnalysis, pushDownVirtualColumnsForLhsExprs);
        }
        if (rewritten == null || !rewritten.isCanPushDown()) {
            return JoinFilterAnalysis.createNoPushdownFilterAnalysis(orFilter);
        } else {
            // noinspection OptionalGetWithoutIsPresent isCanPushDown checks isPresent
            newFilters.add(rewritten.getPushDownFilter().get());
        }
    }
    return new JoinFilterAnalysis(retainRhs, orFilter, Filters.maybeOr(newFilters).orElse(null));
}

Example 20 with Filter

use of org.apache.druid.query.filter.Filter in project druid by druid-io.

the class RhsRewriteCandidates method getRhsRewriteCandidates.

/**
 * Determine candidates for filter rewrites.
 * A candidate is an RHS column that appears in a filter, along with the value being filtered on, plus
 * the joinable clause associated with the table that the RHS column is from.
 *
 * These candidates are redued to filter rewrite correlations.
 *
 * @param normalizedJoinTableClauses
 * @param equiconditions
 * @param joinableClauses
 * @return A set of candidates for filter rewrites.
 */
public static RhsRewriteCandidates getRhsRewriteCandidates(List<Filter> normalizedJoinTableClauses, Equiconditions equiconditions, JoinableClauses joinableClauses) {
    Set<RhsRewriteCandidate> rhsRewriteCandidates = new LinkedHashSet<>();
    for (Filter orClause : normalizedJoinTableClauses) {
        if (Filters.filterMatchesNull(orClause)) {
            continue;
        }
        if (orClause instanceof OrFilter) {
            for (Filter subFilter : ((OrFilter) orClause).getFilters()) {
                Optional<RhsRewriteCandidate> rhsRewriteCandidate = determineRhsRewriteCandidatesForSingleFilter(subFilter, equiconditions, joinableClauses);
                rhsRewriteCandidate.ifPresent(rhsRewriteCandidates::add);
            }
            continue;
        }
        Optional<RhsRewriteCandidate> rhsRewriteCandidate = determineRhsRewriteCandidatesForSingleFilter(orClause, equiconditions, joinableClauses);
        rhsRewriteCandidate.ifPresent(rhsRewriteCandidates::add);
    }
    return new RhsRewriteCandidates(rhsRewriteCandidates);
}