Examples with DbGroupScan org.apache.drill.exec.physical.base.DbGroupScan used on opensource projects

Example 1 with DbGroupScan

use of org.apache.drill.exec.physical.base.DbGroupScan in project drill by apache.

the class DrillRelMdSelectivity method getScanSelectivity.

private Double getScanSelectivity(RelNode rel, RelMetadataQuery mq, RexNode predicate) {
    double ROWCOUNT_UNKNOWN = -1.0;
    GroupScan scan = null;
    PlannerSettings settings = PrelUtil.getPlannerSettings(rel.getCluster().getPlanner());
    final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    if (rel instanceof DrillScanRel) {
        scan = ((DrillScanRel) rel).getGroupScan();
    } else if (rel instanceof ScanPrel) {
        scan = ((ScanPrel) rel).getGroupScan();
    }
    if (scan != null) {
        if (settings.isStatisticsEnabled() && scan instanceof DbGroupScan) {
            double filterRows = ((DbGroupScan) scan).getRowCount(predicate, rel);
            double totalRows = ((DbGroupScan) scan).getRowCount(null, rel);
            if (filterRows != ROWCOUNT_UNKNOWN && totalRows != ROWCOUNT_UNKNOWN && totalRows > 0) {
                return Math.min(1.0, filterRows / totalRows);
            }
        }
    }
    // Do not mess with statistics used for DBGroupScans.
    if (rel instanceof TableScan) {
        if (DrillRelOptUtil.guessRows(rel)) {
            return super.getSelectivity(rel, mq, predicate);
        }
        DrillTable table = Utilities.getDrillTable(rel.getTable());
        try {
            TableMetadata tableMetadata;
            if (table != null && (tableMetadata = table.getGroupScan().getTableMetadata()) != null && TableStatisticsKind.HAS_DESCRIPTIVE_STATISTICS.getValue(tableMetadata)) {
                List<SchemaPath> fieldNames;
                if (rel instanceof DrillScanRelBase) {
                    fieldNames = ((DrillScanRelBase) rel).getGroupScan().getColumns();
                } else {
                    fieldNames = rel.getRowType().getFieldNames().stream().map(SchemaPath::getSimplePath).collect(Collectors.toList());
                }
                return getScanSelectivityInternal(tableMetadata, predicate, fieldNames, rexBuilder);
            }
        } catch (IOException e) {
            super.getSelectivity(rel, mq, predicate);
        }
    }
    return super.getSelectivity(rel, mq, predicate);
}

Example 2 with DbGroupScan

use of org.apache.drill.exec.physical.base.DbGroupScan in project drill by apache.

the class DbScanSortRemovalRule method doOnMatch.

private void doOnMatch(IndexPhysicalPlanCallContext indexContext) {
    Stopwatch indexPlanTimer = Stopwatch.createStarted();
    final PlannerSettings settings = PrelUtil.getPlannerSettings(indexContext.call.getPlanner());
    DbGroupScan groupScan = (DbGroupScan) indexContext.scan.getGroupScan();
    boolean isIndexScan = groupScan.isIndexScan();
    if (!isIndexScan) {
        // This case generates the index scan and removes the sort if possible.
        final IndexCollection indexCollection = groupScan.getSecondaryIndexCollection(indexContext.scan);
        if (indexCollection == null) {
            return;
        }
        if (settings.isStatisticsEnabled()) {
            groupScan.getStatistics().initialize(null, indexContext.scan, indexContext);
        }
        IndexPlanUtils.updateSortExpression(indexContext, indexContext.getSort() != null ? indexContext.getCollation().getFieldCollations() : null);
        IndexSelector selector = new IndexSelector(indexContext);
        for (IndexDescriptor indexDesc : indexCollection) {
            indexDesc.getIndexGroupScan().setStatistics(groupScan.getStatistics());
            FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
            if (IndexPlanUtils.isCoveringIndex(indexContext, functionInfo)) {
                selector.addIndex(indexDesc, true, indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : indexContext.scan.getRowType().getFieldCount());
            }
        }
        IndexProperties idxProp = selector.getBestIndexNoFilter();
        if (idxProp != null) {
            try {
                // generate a covering plan
                CoveringPlanNoFilterGenerator planGen = new CoveringPlanNoFilterGenerator(indexContext, idxProp.getIndexDesc().getFunctionalInfo(), false, settings);
                if (planGen.convertChild() != null) {
                    indexContext.getCall().transformTo(planGen.convertChild());
                } else {
                    logger.debug("Not able to generate index plan in {}", this.getClass().toString());
                }
            } catch (Exception e) {
                logger.warn("Exception while trying to generate indexscan to remove sort", e);
            }
        }
    } else {
        Preconditions.checkNotNull(indexContext.getSort());
        // This case tries to use the already generated index to see if a sort can be removed.
        if (indexContext.scan.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE).getFieldCollations().size() == 0) {
            return;
        }
        try {
            RelNode finalRel = indexContext.scan.copy(indexContext.scan.getTraitSet(), indexContext.scan.getInputs());
            if (indexContext.lowerProject != null) {
                List<RelNode> inputs = Lists.newArrayList();
                inputs.add(finalRel);
                finalRel = indexContext.lowerProject.copy(indexContext.lowerProject.getTraitSet(), inputs);
            }
            finalRel = AbstractIndexPlanGenerator.getSortNode(indexContext, finalRel, true, false, indexContext.exch != null);
            if (finalRel == null) {
                logger.debug("Not able to generate index plan in {}", this.getClass().toString());
                return;
            }
            finalRel = Prule.convert(finalRel, finalRel.getTraitSet().plus(Prel.DRILL_PHYSICAL));
            indexContext.getCall().transformTo(finalRel);
        } catch (Exception e) {
            logger.warn("Exception while trying to use the indexscan to remove the sort", e);
        }
    }
    indexPlanTimer.stop();
    logger.debug("Index Planning took {} ms", indexPlanTimer.elapsed(TimeUnit.MILLISECONDS));
}

Example 3 with DbGroupScan

use of org.apache.drill.exec.physical.base.DbGroupScan in project drill by apache.

the class DrillPushRowKeyJoinToScanRule method doOnMatch.

private void doOnMatch(RelOptRuleCall call, int rowKeyPosition, boolean swapInputs, DrillJoin joinRel, DrillProjectRel upperProjectRel, DrillFilterRel filterRel, DrillProjectRel lowerProjectRel, DrillScanRel scanRel) {
    // Swap the inputs, when necessary (i.e. when the primary-key col is on the right-side of the join)
    logger.debug("Transforming: Swapping of join inputs is required!");
    RelNode right = swapInputs ? joinRel.getLeft() : joinRel.getRight();
    // The join condition is primary-key = COL similarly to PK-FK relationship in relational DBs
    // where primary-key is PK and COL is FK
    List<Integer> leftJoinKeys = ImmutableList.of(rowKeyPosition);
    List<Integer> rightJoinKeys = swapInputs ? joinRel.getLeftKeys() : joinRel.getRightKeys();
    // Create restricted group scan for scanRel and reconstruct the left side of the join.
    DbGroupScan restrictedGroupScan = ((DbGroupScan) scanRel.getGroupScan()).getRestrictedScan(scanRel.getColumns());
    RelNode leftRel = new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet(), scanRel.getTable(), restrictedGroupScan, scanRel.getRowType(), scanRel.getColumns(), scanRel.partitionFilterPushdown());
    // Transform the project/filter rels if present
    if (lowerProjectRel != null) {
        leftRel = lowerProjectRel.copy(lowerProjectRel.getTraitSet(), ImmutableList.of(leftRel));
    }
    if (filterRel != null) {
        leftRel = filterRel.copy(filterRel.getTraitSet(), leftRel, filterRel.getCondition());
    }
    if (upperProjectRel != null) {
        leftRel = upperProjectRel.copy(upperProjectRel.getTraitSet(), ImmutableList.of(leftRel));
    }
    // Create the equi-join condition for the rowkey join
    RexNode joinCondition = RelOptUtil.createEquiJoinCondition(leftRel, leftJoinKeys, right, rightJoinKeys, joinRel.getCluster().getRexBuilder());
    logger.debug("Transforming: LeftKeys={}, LeftRowType={}, RightKeys={}, RightRowType={}", leftJoinKeys, leftRel.getRowType(), rightJoinKeys, right.getRowType());
    RowKeyJoinRel rowKeyJoin = new RowKeyJoinRel(joinRel.getCluster(), joinRel.getTraitSet(), leftRel, right, joinCondition, joinRel.getJoinType(), joinRel instanceof DrillSemiJoinRel);
    logger.info("Transforming: SUCCESS: Register runtime filter pushdown plan (rowkeyjoin)");
    call.transformTo(rowKeyJoin);
}

Example 4 with DbGroupScan

use of org.apache.drill.exec.physical.base.DbGroupScan in project drill by apache.

the class DrillPushRowKeyJoinToScanRule method isRowKeyColumn.

/* Finds whether the given column reference is for the rowkey col(also known as primary-key col).
   * We need to recurse down the operators looking at their references down to the scan
   * to figure out whether the reference is a rowkey col. Projections can rearrange the
   * incoming columns. We also need to handle HepRelVertex/RelSubset while handling the rels.
   */
private static boolean isRowKeyColumn(int index, RelNode rel) {
    RelNode curRel = rel;
    int curIndex = index;
    while (curRel != null && !(curRel instanceof DrillScanRel)) {
        logger.debug("IsRowKeyColumn: Rel={}, RowTypePos={}, RowType={}", curRel.toString(), curIndex, curRel.getRowType().toString());
        if (curRel instanceof HepRelVertex) {
            curRel = ((HepRelVertex) curRel).getCurrentRel();
        } else if (curRel instanceof RelSubset) {
            if (((RelSubset) curRel).getBest() != null) {
                curRel = ((RelSubset) curRel).getBest();
            } else {
                curRel = ((RelSubset) curRel).getOriginal();
            }
        } else {
            RelNode child = null;
            // before recursing down that child rel.
            for (RelNode input : curRel.getInputs()) {
                if (input.getRowType().getFieldList().size() <= curIndex) {
                    curIndex -= input.getRowType().getFieldList().size();
                } else {
                    child = input;
                    break;
                }
            }
            curRel = child;
        }
        // Otherwise, the column index is the `RexInputRef` index.
        if (curRel != null && curRel instanceof DrillProjectRel) {
            List<RexNode> childExprs = curRel.getChildExps();
            if (childExprs != null && childExprs.size() > 0) {
                if (childExprs.get(curIndex) instanceof RexInputRef) {
                    curIndex = ((RexInputRef) childExprs.get(curIndex)).getIndex();
                } else {
                    // Currently do not support expressions on rowkey col. So if an expr is present,
                    // return false
                    logger.debug("IsRowKeyColumn: ABORT: Primary-key EXPR$={}", childExprs.get(curIndex).toString());
                    return false;
                }
            }
        }
    }
    logger.debug("IsRowKeyColumn:Primary-key Col={} ", curRel != null ? curRel.getRowType().getFieldNames().get(curIndex) : "??");
    // Get the primary-key col name from the scan and match with the column being referenced.
    if (curRel != null && curRel instanceof DrillScanRel) {
        if (((DrillScanRel) curRel).getGroupScan() instanceof DbGroupScan) {
            DbGroupScan dbGroupScan = (DbGroupScan) ((DrillScanRel) curRel).getGroupScan();
            String rowKeyName = dbGroupScan.getRowKeyName();
            DbGroupScan restrictedGroupScan = dbGroupScan.getRestrictedScan(((DrillScanRel) curRel).getColumns());
            // Also verify this scan supports restricted groupscans(random seeks)
            if (restrictedGroupScan != null && curRel.getRowType().getFieldNames().get(curIndex).equalsIgnoreCase(rowKeyName)) {
                logger.debug("IsRowKeyColumn: FOUND: Rel={}, RowTypePos={}, RowType={}", curRel.toString(), curIndex, curRel.getRowType().toString());
                return true;
            }
        }
    }
    logger.debug("IsRowKeyColumn: NOT FOUND");
    return false;
}

Example 5 with DbGroupScan

use of org.apache.drill.exec.physical.base.DbGroupScan in project drill by apache.

the class IndexPlanUtils method buildCoveringIndexScan.

public static ScanPrel buildCoveringIndexScan(DrillScanRelBase origScan, IndexGroupScan indexGroupScan, IndexCallContext indexContext, IndexDescriptor indexDesc) {
    FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
    // to record the new (renamed)paths added
    List<SchemaPath> rewrittenPaths = Lists.newArrayList();
    DbGroupScan dbGroupScan = (DbGroupScan) getGroupScan(origScan);
    indexGroupScan.setColumns(rewriteFunctionColumn(dbGroupScan.getColumns(), functionInfo, rewrittenPaths));
    DrillDistributionTrait partition = scanIsPartition(getGroupScan(origScan)) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
    RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, indexContext, functionInfo, rewrittenPaths);
    // add a default collation trait otherwise Calcite runs into a ClassCastException, which at first glance
    // seems like a Calcite bug
    RelTraitSet indexScanTraitSet = origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(RelCollationTraitDef.INSTANCE.getDefault()).plus(partition);
    // condition that the index actually has collation property (e.g hash indexes don't)
    if (indexDesc.getCollation() != null) {
        RelCollation collationTrait = buildCollationCoveringIndexScan(indexDesc, indexContext);
        indexScanTraitSet = indexScanTraitSet.plus(collationTrait);
    }
    ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), indexScanTraitSet, indexGroupScan, newRowType, origScan.getTable());
    return indexScanPrel;
}