Examples with DrillParseContext org.apache.drill.exec.planner.logical.DrillParseContext used on opensource projects

Example 1 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class MongoPushDownFilterForScan method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final ScanPrel scan = (ScanPrel) call.rel(1);
    final FilterPrel filter = (FilterPrel) call.rel(0);
    final RexNode condition = filter.getCondition();
    MongoGroupScan groupScan = (MongoGroupScan) scan.getGroupScan();
    if (groupScan.isFilterPushedDown()) {
        return;
    }
    LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
    MongoFilterBuilder mongoFilterBuilder = new MongoFilterBuilder(groupScan, conditionExp);
    MongoScanSpec newScanSpec = mongoFilterBuilder.parseTree();
    if (newScanSpec == null) {
        // no filter pushdown so nothing to apply.
        return;
    }
    MongoGroupScan newGroupsScan = null;
    try {
        newGroupsScan = new MongoGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), newScanSpec, groupScan.getColumns());
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        throw new DrillRuntimeException(e.getMessage(), e);
    }
    newGroupsScan.setFilterPushedDown(true);
    final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
    if (mongoFilterBuilder.isAllExpressionsConverted()) {
        /*
       * Since we could convert the entire filter condition expression into an
       * Mongo filter, we can eliminate the filter operator altogether.
       */
        call.transformTo(newScanPrel);
    } else {
        call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of((RelNode) newScanPrel)));
    }
}

Example 2 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by axbaretto.

the class HBasePushFilterIntoScan method doPushFilterToScan.

protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final HBaseGroupScan groupScan, final RexNode condition) {
    final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
    final HBaseFilterBuilder hbaseFilterBuilder = new HBaseFilterBuilder(groupScan, conditionExp);
    final HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree();
    if (newScanSpec == null) {
        // no filter pushdown ==> No transformation.
        return;
    }
    final HBaseGroupScan newGroupsScan = new HBaseGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), newScanSpec, groupScan.getColumns());
    newGroupsScan.setFilterPushedDown(true);
    final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
    // Depending on whether is a project in the middle, assign either scan or copy of project to childRel.
    final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode) newScanPrel));
    ;
    if (hbaseFilterBuilder.isAllExpressionsConverted()) {
        /*
         * Since we could convert the entire filter condition expression into an HBase filter,
         * we can eliminate the filter operator altogether.
         */
        call.transformTo(childRel);
    } else {
        call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel)));
    }
}

Example 3 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by axbaretto.

the class FlattenPrel method getPhysicalOperator.

@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
    Prel child = (Prel) this.getInput();
    PhysicalOperator childPOP = child.getPhysicalOperator(creator);
    FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext(PrelUtil.getSettings(getCluster()))));
    return creator.addMetadata(this, f);
}

Example 4 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by axbaretto.

the class ProjectAllowDupPrel method getPhysicalOperator.

@Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
    Prel child = (Prel) this.getInput();
    PhysicalOperator childPOP = child.getPhysicalOperator(creator);
    Project p = new Project(this.getProjectExpressions(new DrillParseContext(PrelUtil.getSettings(getCluster()))), childPOP);
    return creator.addMetadata(this, p);
}

Example 5 with DrillParseContext

use of org.apache.drill.exec.planner.logical.DrillParseContext in project drill by apache.

the class FilePushDownFilter method doOnMatch.

protected void doOnMatch(RelOptRuleCall call, FilterPrel filter, ProjectPrel project, ScanPrel scan) {
    AbstractGroupScanWithMetadata<?> groupScan = (AbstractGroupScanWithMetadata<?>) scan.getGroupScan();
    if (groupScan.getFilter() != null && !groupScan.getFilter().equals(ValueExpressions.BooleanExpression.TRUE)) {
        return;
    }
    RexNode condition;
    if (project == null) {
        condition = filter.getCondition();
    } else {
        // get the filter as if it were below the projection.
        condition = RelOptUtil.pushPastProject(filter.getCondition(), project);
    }
    if (condition == null || condition.isAlwaysTrue()) {
        return;
    }
    // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
    // then we could not pushed down. Otherwise, it's qualified to be pushed down.
    // Limits the number of nodes that can be created out of the conversion to avoid
    // exponential growth of nodes count and further OOM
    final List<RexNode> predList = RelOptUtil.conjunctions(RexUtil.toCnf(filter.getCluster().getRexBuilder(), 100, condition));
    final List<RexNode> qualifiedPredList = new ArrayList<>();
    // list of predicates which cannot be converted to filter predicate
    List<RexNode> nonConvertedPredList = new ArrayList<>();
    for (RexNode pred : predList) {
        if (DrillRelOptUtil.findOperators(pred, Collections.emptyList(), BANNED_OPERATORS) == null) {
            LogicalExpression drillPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, pred);
            // checks whether predicate may be used for filter pushdown
            FilterPredicate<?> filterPredicate = groupScan.getFilterPredicate(drillPredicate, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions(), false);
            // to build filter with them
            if (filterPredicate == null) {
                nonConvertedPredList.add(pred);
            }
            qualifiedPredList.add(pred);
        } else {
            nonConvertedPredList.add(pred);
        }
    }
    final RexNode qualifiedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
    if (qualifiedPred == null) {
        return;
    }
    LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifiedPred);
    // Default - pass the original filter expr to (potentially) be used at run-time
    // later may remove or set to another filter (see below)
    groupScan.setFilterForRuntime(conditionExp, optimizerContext);
    Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
    AbstractGroupScanWithMetadata<?> newGroupScan = groupScan.applyFilter(conditionExp, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions());
    if (timer != null) {
        logger.debug("Took {} ms to apply filter. ", timer.elapsed(TimeUnit.MILLISECONDS));
        timer.stop();
    }
    // fully match the filter for the case when row group pruning did not happen.
    if (newGroupScan == null) {
        if (groupScan.isMatchAllMetadata()) {
            RelNode child = project == null ? scan : project;
            // but row group pruning did not happen, remove the filter.
            if (nonConvertedPredList.isEmpty()) {
                // disable the original filter expr (i.e. don't use it at run-time)
                groupScan.setFilterForRuntime(null, optimizerContext);
                call.transformTo(child);
            } else if (nonConvertedPredList.size() == predList.size()) {
                // None of the predicates participated in filter pushdown.
                return;
            } else {
                // If some of the predicates weren't used in the filter, creates new filter with them
                // on top of current scan. Excludes the case when all predicates weren't used in the filter.
                Filter theNewFilter = filter.copy(filter.getTraitSet(), child, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
                LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theNewFilter.getCondition());
                // pass the new filter expr to (potentialy) be used at run-time
                groupScan.setFilterForRuntime(filterPredicate, optimizerContext);
                // Replace the child with the new filter on top of the child/scan
                call.transformTo(theNewFilter);
            }
        }
        return;
    }
    RelNode newNode = new ScanPrel(scan.getCluster(), scan.getTraitSet(), newGroupScan, scan.getRowType(), scan.getTable());
    if (project != null) {
        newNode = project.copy(project.getTraitSet(), Collections.singletonList(newNode));
    }
    if (newGroupScan.isMatchAllMetadata()) {
        // creates filter from the expressions which can't be pushed to the scan
        if (!nonConvertedPredList.isEmpty()) {
            Filter theFilterRel = filter.copy(filter.getTraitSet(), newNode, RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), nonConvertedPredList, true));
            LogicalExpression filterPredicate = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, theFilterRel.getCondition());
            // pass the new filter expr to (potentialy) be used at run-time
            newGroupScan.setFilterForRuntime(filterPredicate, optimizerContext);
            // replace the new node with the new filter on top of that new node
            newNode = theFilterRel;
        }
        call.transformTo(newNode);
        return;
    }
    final RelNode newFilter = filter.copy(filter.getTraitSet(), Collections.singletonList(newNode));
    call.transformTo(newFilter);
}