Examples with PlannerSettings org.apache.drill.exec.planner.physical.PlannerSettings used on opensource projects

Example 6 with PlannerSettings

use of org.apache.drill.exec.planner.physical.PlannerSettings in project drill by apache.

the class HivePushPartitionFilterIntoScan method getFilterOnProject.

public static final StoragePluginOptimizerRule getFilterOnProject(OptimizerRulesContext optimizerRulesContext, final String defaultPartitionValue) {
    return new PruneScanRule(RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), "HivePushPartitionFilterIntoScan:Filter_On_Project_Hive", optimizerRulesContext) {

        @Override
        public PartitionDescriptor getPartitionDescriptor(PlannerSettings settings, TableScan scanRel) {
            return new HivePartitionDescriptor(settings, (DrillScanRel) scanRel, getOptimizerRulesContext().getManagedBuffer(), defaultPartitionValue);
        }

        @Override
        public boolean matches(RelOptRuleCall call) {
            final DrillScanRel scan = (DrillScanRel) call.rel(2);
            GroupScan groupScan = scan.getGroupScan();
            // this rule is applicable only for Hive based partition pruning
            if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) {
                return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown();
            } else {
                return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
            }
        }

        @Override
        public void onMatch(RelOptRuleCall call) {
            final DrillFilterRel filterRel = call.rel(0);
            final DrillProjectRel projectRel = call.rel(1);
            final DrillScanRel scanRel = call.rel(2);
            doOnMatch(call, filterRel, projectRel, scanRel);
        }
    };
}

Example 7 with PlannerSettings

use of org.apache.drill.exec.planner.physical.PlannerSettings in project drill by apache.

the class HivePushPartitionFilterIntoScan method getFilterOnScan.

public static final StoragePluginOptimizerRule getFilterOnScan(OptimizerRulesContext optimizerRulesContext, final String defaultPartitionValue) {
    return new PruneScanRule(RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)), "HivePushPartitionFilterIntoScan:Filter_On_Scan_Hive", optimizerRulesContext) {

        @Override
        public PartitionDescriptor getPartitionDescriptor(PlannerSettings settings, TableScan scanRel) {
            return new HivePartitionDescriptor(settings, (DrillScanRel) scanRel, getOptimizerRulesContext().getManagedBuffer(), defaultPartitionValue);
        }

        @Override
        public boolean matches(RelOptRuleCall call) {
            final DrillScanRel scan = (DrillScanRel) call.rel(1);
            GroupScan groupScan = scan.getGroupScan();
            // this rule is applicable only for Hive based partition pruning
            if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) {
                return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown();
            } else {
                return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
            }
        }

        @Override
        public void onMatch(RelOptRuleCall call) {
            final DrillFilterRel filterRel = call.rel(0);
            final DrillScanRel scanRel = call.rel(1);
            doOnMatch(call, filterRel, null, scanRel);
        }
    };
}

Example 8 with PlannerSettings

use of org.apache.drill.exec.planner.physical.PlannerSettings in project drill by apache.

the class DrillRelMdDistinctRowCount method getDistinctRowCountInternal.

private Double getDistinctRowCountInternal(DrillJoinRelBase joinRel, RelMetadataQuery mq, ImmutableBitSet groupKey, RexNode predicate) {
    if (DrillRelOptUtil.guessRows(joinRel)) {
        return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
    }
    // Assume NDV is unaffected by the join when groupKey comes from one side of the join
    // Alleviates NDV over-estimates
    ImmutableBitSet.Builder leftMask = ImmutableBitSet.builder();
    ImmutableBitSet.Builder rightMask = ImmutableBitSet.builder();
    JoinRelType joinType = joinRel.getJoinType();
    RelNode left = joinRel.getInputs().get(0);
    RelNode right = joinRel.getInputs().get(1);
    RelMdUtil.setLeftRightBitmaps(groupKey, leftMask, rightMask, left.getRowType().getFieldCount());
    RexNode leftPred = null;
    RexNode rightPred = null;
    // Identify predicates which can be pushed onto the left and right sides of the join
    if (predicate != null) {
        List<RexNode> leftFilters = new ArrayList<>();
        List<RexNode> rightFilters = new ArrayList<>();
        List<RexNode> joinFilters = new ArrayList();
        List<RexNode> predList = RelOptUtil.conjunctions(predicate);
        RelOptUtil.classifyFilters(joinRel, predList, joinType, joinType == JoinRelType.INNER, !joinType.generatesNullsOnLeft(), !joinType.generatesNullsOnRight(), joinFilters, leftFilters, rightFilters);
        RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
        leftPred = RexUtil.composeConjunction(rexBuilder, leftFilters, true);
        rightPred = RexUtil.composeConjunction(rexBuilder, rightFilters, true);
    }
    double distRowCount = 1;
    int gbyCols = 0;
    PlannerSettings plannerSettings = PrelUtil.getPlannerSettings(joinRel.getCluster().getPlanner());
    /*
     * The NDV for a multi-column GBY key past a join is determined as follows:
     * GBY(s1, s2, s3) = CNDV(s1)*CNDV(s2)*CNDV(s3)
     * where CNDV is determined as follows:
     * A) If sX is present as a join column (sX = tX) CNDV(sX) = MIN(NDV(sX), NDV(tX)) where X =1, 2, 3, etc
     * B) Otherwise, based on independence assumption CNDV(sX) = NDV(sX)
     */
    Set<ImmutableBitSet> joinFiltersSet = new HashSet<>();
    for (RexNode filter : RelOptUtil.conjunctions(joinRel.getCondition())) {
        final RelOptUtil.InputFinder inputFinder = RelOptUtil.InputFinder.analyze(filter);
        joinFiltersSet.add(inputFinder.inputBitSet.build());
    }
    for (int idx = 0; idx < groupKey.length(); idx++) {
        if (groupKey.get(idx)) {
            // GBY key is present in some filter - now try options A) and B) as described above
            double ndvSGby = Double.MAX_VALUE;
            Double ndv;
            boolean presentInFilter = false;
            ImmutableBitSet sGby = getSingleGbyKey(groupKey, idx);
            if (sGby != null) {
                // If we see any NULL ndv i.e. cant process ..we bail out!
                for (ImmutableBitSet jFilter : joinFiltersSet) {
                    if (jFilter.contains(sGby)) {
                        presentInFilter = true;
                        // Found join condition containing this GBY key. Pick min NDV across all columns in this join
                        for (int fidx : jFilter) {
                            if (fidx < left.getRowType().getFieldCount()) {
                                ndv = mq.getDistinctRowCount(left, ImmutableBitSet.of(fidx), leftPred);
                                if (ndv == null) {
                                    return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
                                }
                                ndvSGby = Math.min(ndvSGby, ndv);
                            } else {
                                ndv = mq.getDistinctRowCount(right, ImmutableBitSet.of(fidx - left.getRowType().getFieldCount()), rightPred);
                                if (ndv == null) {
                                    return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
                                }
                                ndvSGby = Math.min(ndvSGby, ndv);
                            }
                        }
                        break;
                    }
                }
                // Did not find it in any join condition(s)
                if (!presentInFilter) {
                    for (int sidx : sGby) {
                        if (sidx < left.getRowType().getFieldCount()) {
                            ndv = mq.getDistinctRowCount(left, ImmutableBitSet.of(sidx), leftPred);
                            if (ndv == null) {
                                return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
                            }
                            ndvSGby = ndv;
                        } else {
                            ndv = mq.getDistinctRowCount(right, ImmutableBitSet.of(sidx - left.getRowType().getFieldCount()), rightPred);
                            if (ndv == null) {
                                return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
                            }
                            ndvSGby = ndv;
                        }
                    }
                }
                ++gbyCols;
                // Multiply NDV(s) of different GBY cols to determine the overall NDV
                distRowCount *= ndvSGby;
            }
        }
    }
    if (gbyCols > 1) {
        // Scale with multi-col NDV factor if more than one GBY cols were found
        distRowCount *= plannerSettings.getStatisticsMultiColNdvAdjustmentFactor();
    }
    double joinRowCount = mq.getRowCount(joinRel);
    // Cap NDV to join row count
    distRowCount = Math.min(distRowCount, joinRowCount);
    return RelMdUtil.numDistinctVals(distRowCount, joinRowCount);
}

Example 9 with PlannerSettings

use of org.apache.drill.exec.planner.physical.PlannerSettings in project drill by apache.

the class DrillRelMdSelectivity method getScanSelectivity.

private Double getScanSelectivity(RelNode rel, RelMetadataQuery mq, RexNode predicate) {
    double ROWCOUNT_UNKNOWN = -1.0;
    GroupScan scan = null;
    PlannerSettings settings = PrelUtil.getPlannerSettings(rel.getCluster().getPlanner());
    final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    if (rel instanceof DrillScanRel) {
        scan = ((DrillScanRel) rel).getGroupScan();
    } else if (rel instanceof ScanPrel) {
        scan = ((ScanPrel) rel).getGroupScan();
    }
    if (scan != null) {
        if (settings.isStatisticsEnabled() && scan instanceof DbGroupScan) {
            double filterRows = ((DbGroupScan) scan).getRowCount(predicate, rel);
            double totalRows = ((DbGroupScan) scan).getRowCount(null, rel);
            if (filterRows != ROWCOUNT_UNKNOWN && totalRows != ROWCOUNT_UNKNOWN && totalRows > 0) {
                return Math.min(1.0, filterRows / totalRows);
            }
        }
    }
    // Do not mess with statistics used for DBGroupScans.
    if (rel instanceof TableScan) {
        if (DrillRelOptUtil.guessRows(rel)) {
            return super.getSelectivity(rel, mq, predicate);
        }
        DrillTable table = Utilities.getDrillTable(rel.getTable());
        try {
            TableMetadata tableMetadata;
            if (table != null && (tableMetadata = table.getGroupScan().getTableMetadata()) != null && TableStatisticsKind.HAS_DESCRIPTIVE_STATISTICS.getValue(tableMetadata)) {
                List<SchemaPath> fieldNames;
                if (rel instanceof DrillScanRelBase) {
                    fieldNames = ((DrillScanRelBase) rel).getGroupScan().getColumns();
                } else {
                    fieldNames = rel.getRowType().getFieldNames().stream().map(SchemaPath::getSimplePath).collect(Collectors.toList());
                }
                return getScanSelectivityInternal(tableMetadata, predicate, fieldNames, rexBuilder);
            }
        } catch (IOException e) {
            super.getSelectivity(rel, mq, predicate);
        }
    }
    return super.getSelectivity(rel, mq, predicate);
}

Example 10 with PlannerSettings

use of org.apache.drill.exec.planner.physical.PlannerSettings in project drill by apache.

the class DbScanSortRemovalRule method doOnMatch.

private void doOnMatch(IndexPhysicalPlanCallContext indexContext) {
    Stopwatch indexPlanTimer = Stopwatch.createStarted();
    final PlannerSettings settings = PrelUtil.getPlannerSettings(indexContext.call.getPlanner());
    DbGroupScan groupScan = (DbGroupScan) indexContext.scan.getGroupScan();
    boolean isIndexScan = groupScan.isIndexScan();
    if (!isIndexScan) {
        // This case generates the index scan and removes the sort if possible.
        final IndexCollection indexCollection = groupScan.getSecondaryIndexCollection(indexContext.scan);
        if (indexCollection == null) {
            return;
        }
        if (settings.isStatisticsEnabled()) {
            groupScan.getStatistics().initialize(null, indexContext.scan, indexContext);
        }
        IndexPlanUtils.updateSortExpression(indexContext, indexContext.getSort() != null ? indexContext.getCollation().getFieldCollations() : null);
        IndexSelector selector = new IndexSelector(indexContext);
        for (IndexDescriptor indexDesc : indexCollection) {
            indexDesc.getIndexGroupScan().setStatistics(groupScan.getStatistics());
            FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
            if (IndexPlanUtils.isCoveringIndex(indexContext, functionInfo)) {
                selector.addIndex(indexDesc, true, indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : indexContext.scan.getRowType().getFieldCount());
            }
        }
        IndexProperties idxProp = selector.getBestIndexNoFilter();
        if (idxProp != null) {
            try {
                // generate a covering plan
                CoveringPlanNoFilterGenerator planGen = new CoveringPlanNoFilterGenerator(indexContext, idxProp.getIndexDesc().getFunctionalInfo(), false, settings);
                if (planGen.convertChild() != null) {
                    indexContext.getCall().transformTo(planGen.convertChild());
                } else {
                    logger.debug("Not able to generate index plan in {}", this.getClass().toString());
                }
            } catch (Exception e) {
                logger.warn("Exception while trying to generate indexscan to remove sort", e);
            }
        }
    } else {
        Preconditions.checkNotNull(indexContext.getSort());
        // This case tries to use the already generated index to see if a sort can be removed.
        if (indexContext.scan.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE).getFieldCollations().size() == 0) {
            return;
        }
        try {
            RelNode finalRel = indexContext.scan.copy(indexContext.scan.getTraitSet(), indexContext.scan.getInputs());
            if (indexContext.lowerProject != null) {
                List<RelNode> inputs = Lists.newArrayList();
                inputs.add(finalRel);
                finalRel = indexContext.lowerProject.copy(indexContext.lowerProject.getTraitSet(), inputs);
            }
            finalRel = AbstractIndexPlanGenerator.getSortNode(indexContext, finalRel, true, false, indexContext.exch != null);
            if (finalRel == null) {
                logger.debug("Not able to generate index plan in {}", this.getClass().toString());
                return;
            }
            finalRel = Prule.convert(finalRel, finalRel.getTraitSet().plus(Prel.DRILL_PHYSICAL));
            indexContext.getCall().transformTo(finalRel);
        } catch (Exception e) {
            logger.warn("Exception while trying to use the indexscan to remove the sort", e);
        }
    }
    indexPlanTimer.stop();
    logger.debug("Index Planning took {} ms", indexPlanTimer.elapsed(TimeUnit.MILLISECONDS));
}