Examples with Filter org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter used on opensource projects

Example 36 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project hive by apache.

the class HivePreFilteringRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Filter filter = call.rel(0);
    // 0. Register that we have visited this operator in this rule
    HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
    if (registry != null) {
        registry.registerVisited(this, filter);
    }
    final RexBuilder rexBuilder = filter.getCluster().getRexBuilder();
    // 1. Recompose filter possibly by pulling out common elements from DNF
    // expressions
    RexNode topFilterCondition = RexUtil.pullFactors(rexBuilder, filter.getCondition());
    // 2. We extract possible candidates to be pushed down
    List<RexNode> operandsToPushDown = new ArrayList<>();
    List<RexNode> deterministicExprs = new ArrayList<>();
    List<RexNode> nonDeterministicExprs = new ArrayList<>();
    switch(topFilterCondition.getKind()) {
        case AND:
            ImmutableList<RexNode> operands = RexUtil.flattenAnd(((RexCall) topFilterCondition).getOperands());
            Set<String> operandsToPushDownDigest = new HashSet<String>();
            List<RexNode> extractedCommonOperands = null;
            for (RexNode operand : operands) {
                if (operand.getKind() == SqlKind.OR) {
                    extractedCommonOperands = extractCommonOperands(rexBuilder, filter.getInput(), operand, maxCNFNodeCount);
                    for (RexNode extractedExpr : extractedCommonOperands) {
                        if (operandsToPushDownDigest.add(extractedExpr.toString())) {
                            operandsToPushDown.add(extractedExpr);
                        }
                    }
                }
                // elements of DNF/CNF & extract more deterministic pieces out.
                if (HiveCalciteUtil.isDeterministic(operand)) {
                    deterministicExprs.add(operand);
                } else {
                    nonDeterministicExprs.add(operand);
                }
            }
            // NOTE: Hive by convention doesn't pushdown non deterministic expressions
            if (nonDeterministicExprs.size() > 0) {
                for (RexNode expr : deterministicExprs) {
                    if (!operandsToPushDownDigest.contains(expr.toString())) {
                        operandsToPushDown.add(expr);
                        operandsToPushDownDigest.add(expr.toString());
                    }
                }
                topFilterCondition = RexUtil.pullFactors(rexBuilder, RexUtil.composeConjunction(rexBuilder, nonDeterministicExprs, false));
            }
            break;
        case OR:
            operandsToPushDown = extractCommonOperands(rexBuilder, filter.getInput(), topFilterCondition, maxCNFNodeCount);
            break;
        default:
            return;
    }
    // 2. If we did not generate anything for the new predicate, we bail out
    if (operandsToPushDown.isEmpty()) {
        return;
    }
    // 3. If the new conjuncts are already present in the plan, we bail out
    final List<RexNode> newConjuncts = HiveCalciteUtil.getPredsNotPushedAlready(filter.getInput(), operandsToPushDown);
    RexNode newPredicate = RexUtil.composeConjunction(rexBuilder, newConjuncts, false);
    if (newPredicate.isAlwaysTrue()) {
        return;
    }
    // 4. Otherwise, we create a new condition
    final RexNode newChildFilterCondition = RexUtil.pullFactors(rexBuilder, newPredicate);
    // 5. We create the new filter that might be pushed down
    RelNode newChildFilter = filterFactory.createFilter(filter.getInput(), newChildFilterCondition);
    RelNode newTopFilter = filterFactory.createFilter(newChildFilter, topFilterCondition);
    // 6. We register both so we do not fire the rule on them again
    if (registry != null) {
        registry.registerVisited(this, newChildFilter);
        registry.registerVisited(this, newTopFilter);
    }
    call.transformTo(newTopFilter);
}

Example 37 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project hive by apache.

the class HiveFilterSetOpTransposeRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    Filter filterRel = call.rel(0);
    RexNode condition = filterRel.getCondition();
    if (!HiveCalciteUtil.isDeterministic(condition)) {
        return false;
    }
    return super.matches(call);
}

Example 38 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project hive by apache.

the class HiveFilterSetOpTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
// implement RelOptRule
// We override the rule in order to do union all branch elimination
public void onMatch(RelOptRuleCall call) {
    Filter filterRel = call.rel(0);
    SetOp setOp = call.rel(1);
    RexNode condition = filterRel.getCondition();
    // create filters on top of each setop child, modifying the filter
    // condition to reference each setop child
    RexBuilder rexBuilder = filterRel.getCluster().getRexBuilder();
    final RelBuilder relBuilder = call.builder();
    List<RelDataTypeField> origFields = setOp.getRowType().getFieldList();
    int[] adjustments = new int[origFields.size()];
    final List<RelNode> newSetOpInputs = new ArrayList<>();
    RelNode lastInput = null;
    for (int index = 0; index < setOp.getInputs().size(); index++) {
        RelNode input = setOp.getInput(index);
        RexNode newCondition = condition.accept(new RelOptUtil.RexInputConverter(rexBuilder, origFields, input.getRowType().getFieldList(), adjustments));
        if (setOp instanceof Union && setOp.all) {
            final RelMetadataQuery mq = call.getMetadataQuery();
            final RelOptPredicateList predicates = mq.getPulledUpPredicates(input);
            if (predicates != null) {
                ImmutableList.Builder<RexNode> listBuilder = ImmutableList.builder();
                listBuilder.addAll(predicates.pulledUpPredicates);
                listBuilder.add(newCondition);
                RexExecutor executor = Util.first(filterRel.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR);
                final RexSimplify simplify = new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, executor);
                final RexNode cond = RexUtil.composeConjunction(rexBuilder, listBuilder.build());
                final RexNode x = simplify.simplifyUnknownAs(cond, RexUnknownAs.FALSE);
                if (x.isAlwaysFalse()) {
                    // branch so it won't read any data.
                    if (index == setOp.getInputs().size() - 1) {
                        lastInput = relBuilder.push(input).filter(newCondition).build();
                    }
                    // remove this branch
                    continue;
                }
            }
        }
        newSetOpInputs.add(relBuilder.push(input).filter(newCondition).build());
    }
    if (newSetOpInputs.size() > 1) {
        // create a new setop whose children are the filters created above
        SetOp newSetOp = setOp.copy(setOp.getTraitSet(), newSetOpInputs);
        call.transformTo(newSetOp);
    } else {
        // We have to keep at least a branch before we support empty values() in Hive
        RelNode result = newSetOpInputs.size() == 1 ? newSetOpInputs.get(0) : lastInput;
        call.transformTo(relBuilder.push(result).convert(filterRel.getRowType(), false).build());
    }
}

Example 39 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project hive by apache.

the class HiveFilterAggregateTransposeRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Filter filterRel = call.rel(0);
    RexNode condition = filterRel.getCondition();
    if (!HiveCalciteUtil.isDeterministic(condition)) {
        return false;
    }
    return super.matches(call);
}

Example 40 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project hive by apache.

the class HiveFilterProjectTransposeRule method onMatch.

public void onMatch(RelOptRuleCall call) {
    final Filter filter = call.rel(0);
    final Project origproject = call.rel(1);
    RexNode filterCondToPushBelowProj = filter.getCondition();
    RexNode unPushedFilCondAboveProj = null;
    if (RexUtil.containsCorrelation(filterCondToPushBelowProj)) {
        // Correlate from being de-correlated.
        return;
    }
    if (RexOver.containsOver(origproject.getProjects(), null)) {
        RexNode origFilterCond = filterCondToPushBelowProj;
        filterCondToPushBelowProj = null;
        if (pushThroughWindowing) {
            Set<Integer> commonPartitionKeys = getCommonPartitionCols(origproject.getProjects());
            List<RexNode> newPartKeyFilConds = new ArrayList<>();
            List<RexNode> unpushedFilConds = new ArrayList<>();
            // from t1 where value < 10)t1)t2
            if (!commonPartitionKeys.isEmpty()) {
                for (RexNode ce : RelOptUtil.conjunctions(origFilterCond)) {
                    RexNode newCondition = RelOptUtil.pushPastProject(ce, origproject);
                    if (HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition, commonPartitionKeys)) {
                        newPartKeyFilConds.add(ce);
                    } else {
                        unpushedFilConds.add(ce);
                    }
                }
                if (!newPartKeyFilConds.isEmpty()) {
                    filterCondToPushBelowProj = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), newPartKeyFilConds, true);
                }
                if (!unpushedFilConds.isEmpty()) {
                    unPushedFilCondAboveProj = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), unpushedFilConds, true);
                }
            }
        }
    }
    if (filterCondToPushBelowProj != null && !isRedundantIsNotNull(origproject, filterCondToPushBelowProj)) {
        RelNode newProjRel = getNewProject(filterCondToPushBelowProj, unPushedFilCondAboveProj, origproject, filter.getCluster().getTypeFactory(), call.builder());
        call.transformTo(newProjRel);
    }
}