Examples with AbstractGroupScanWithMetadata org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata used on opensource projects

Example 1 with AbstractGroupScanWithMetadata

use of org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata in project drill by apache.

the class FilePushDownFilter method doOnMatch.

protected void doOnMatch(RelOptRuleCall call, FilterPrel filter, ProjectPrel project, ScanPrel scan) {
    AbstractGroupScanWithMetadata<?> groupScan = (AbstractGroupScanWithMetadata<?>) scan.getGroupScan();
    if (groupScan.getFilter() != null && !groupScan.getFilter().equals(ValueExpressions.BooleanExpression.TRUE)) {
        return;
    }
    RexNode condition;
    if (project == null) {
        condition = filter.getCondition();
    } else {
        // get the filter as if it were below the projection.
        condition = RelOptUtil.pushPastProject(filter.getCondition(), project);
    }
    if (condition == null || condition.isAlwaysTrue()) {
        return;
    }
    // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
    // then we could not pushed down. Otherwise, it's qualified to be pushed down.
    // Limits the number of nodes that can be created out of the conversion to avoid
    // exponential growth of nodes count and further OOM
    final List<RexNode> predList = RelOptUtil.conjunctions(RexUtil.toCnf(filter.getCluster().getRexBuilder(), 100, condition));
    final List<RexNode> qualifiedPredList = new ArrayList<>();
    // list of predicates which cannot be converted to filter predicate
    List<RexNode> nonConvertedPredList = new ArrayList<>();
    for (RexNode pred : predList) {
        if (DrillRelOptUtil.findOperators(pred, Collections.emptyList(), BANNED_OPERATORS) == null) {
            LogicalExpression drillPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, pred);
            // checks whether predicate may be used for filter pushdown
            FilterPredicate<?> filterPredicate = groupScan.getFilterPredicate(drillPredicate, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions(), false);
            // to build filter with them
            if (filterPredicate == null) {
                nonConvertedPredList.add(pred);
            }
            qualifiedPredList.add(pred);
        } else {
            nonConvertedPredList.add(pred);
        }
    }
    final RexNode qualifiedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
    if (qualifiedPred == null) {
        return;
    }
    LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifiedPred);
    // Default - pass the original filter expr to (potentially) be used at run-time
    // later may remove or set to another filter (see below)
    groupScan.setFilterForRuntime(conditionExp, optimizerContext);
    Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
    AbstractGroupScanWithMetadata<?> newGroupScan = groupScan.applyFilter(conditionExp, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions());
    if (timer != null) {
        logger.debug("Took {} ms to apply filter. ", timer.elapsed(TimeUnit.MILLISECONDS));
        timer.stop();
    }
    // fully match the filter for the case when row group pruning did not happen.
    if (newGroupScan == null) {
        if (groupScan.isMatchAllMetadata()) {
            RelNode child = project == null ? scan : project;
            // but row group pruning did not happen, remove the filter.
            if (nonConvertedPredList.isEmpty()) {
                // disable the original filter expr (i.e. don't use it at run-time)
                groupScan.setFilterForRuntime(null, optimizerContext);
                call.transformTo(child);
            } else if (nonConvertedPredList.size() == predList.size()) {
                // None of the predicates participated in filter pushdown.
                return;
            } else {
                // If some of the predicates weren't used in the filter, creates new filter with them
                // on top of current scan. Excludes the case when all predicates weren't used in the filter.
                Filter theNewFilter = filter.copy(filter.getTraitSet(), child, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
                LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theNewFilter.getCondition());
                // pass the new filter expr to (potentialy) be used at run-time
                groupScan.setFilterForRuntime(filterPredicate, optimizerContext);
                // Replace the child with the new filter on top of the child/scan
                call.transformTo(theNewFilter);
            }
        }
        return;
    }
    RelNode newNode = new ScanPrel(scan.getCluster(), scan.getTraitSet(), newGroupScan, scan.getRowType(), scan.getTable());
    if (project != null) {
        newNode = project.copy(project.getTraitSet(), Collections.singletonList(newNode));
    }
    if (newGroupScan.isMatchAllMetadata()) {
        // creates filter from the expressions which can't be pushed to the scan
        if (!nonConvertedPredList.isEmpty()) {
            Filter theFilterRel = filter.copy(filter.getTraitSet(), newNode, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
            LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theFilterRel.getCondition());
            // pass the new filter expr to (potentialy) be used at run-time
            newGroupScan.setFilterForRuntime(filterPredicate, optimizerContext);
            // replace the new node with the new filter on top of that new node
            newNode = theFilterRel;
        }
        call.transformTo(newNode);
        return;
    }
    final RelNode newFilter = filter.copy(filter.getTraitSet(), Collections.singletonList(newNode));
    call.transformTo(newFilter);
}