Examples with GroupScan org.apache.drill.exec.physical.base.GroupScan used on opensource projects

Example 21 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by apache.

the class KafkaPushDownFilterIntoScan method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final ScanPrel scan = call.rel(1);
    final FilterPrel filter = call.rel(0);
    final RexNode condition = filter.getCondition();
    LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
    KafkaGroupScan groupScan = (KafkaGroupScan) scan.getGroupScan();
    if (logger.isDebugEnabled()) {
        logger.debug("Partitions ScanSpec before push down: {}", groupScan.getPartitionScanSpecList());
    }
    List<KafkaPartitionScanSpec> newScanSpec;
    try (KafkaPartitionScanSpecBuilder builder = new KafkaPartitionScanSpecBuilder(groupScan, conditionExp)) {
        newScanSpec = builder.parseTree();
    }
    // No pushdown
    if (newScanSpec == null) {
        return;
    }
    logger.debug("Partitions ScanSpec after pushdown: {}", newScanSpec);
    GroupScan newGroupScan = groupScan.cloneWithNewSpec(newScanSpec);
    final ScanPrel newScanPrel = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupScan, scan.getRowType(), scan.getTable());
    call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(newScanPrel)));
}

Example 22 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by apache.

the class ParquetPartitionDescriptor method createTableScan.

@Override
public TableScan createTableScan(List<PartitionLocation> newPartitionLocation, Path cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws Exception {
    List<Path> newFiles = new ArrayList<>();
    for (final PartitionLocation location : newPartitionLocation) {
        newFiles.add(location.getEntirePartitionLocation());
    }
    final GroupScan newGroupScan = createNewGroupScan(newFiles, cacheFileRoot, wasAllPartitionsPruned, metaContext);
    if (newGroupScan == null) {
        logger.warn("Unable to create new group scan, returning original table scan.");
        return scanRel;
    }
    return new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), scanRel.getTable(), newGroupScan, scanRel.getRowType(), scanRel.getColumns(), true);
}

Example 23 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by apache.

the class ConvertMetadataAggregateToDirectScanRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    MetadataAggRel agg = call.rel(0);
    DrillScanRel scan = call.rel(1);
    GroupScan oldGrpScan = scan.getGroupScan();
    PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
    // Only apply the rule for parquet group scan and for the case when required column metadata is present
    if (!(oldGrpScan instanceof ParquetGroupScan) || (oldGrpScan.getTableMetadata().getInterestingColumns() != null && !oldGrpScan.getTableMetadata().getInterestingColumns().containsAll(agg.getContext().interestingColumns()))) {
        return;
    }
    try {
        DirectGroupScan directScan = buildDirectScan(agg.getContext().interestingColumns(), scan, settings);
        if (directScan == null) {
            logger.warn("Unable to use parquet metadata for ANALYZE since some required metadata is absent within parquet metadata");
            return;
        }
        RelNode converted = new DrillDirectScanRel(scan.getCluster(), scan.getTraitSet().plus(DrillRel.DRILL_LOGICAL), directScan, scan.getRowType());
        if (agg.getContext().metadataLevel() != MetadataType.ROW_GROUP) {
            MetadataAggregateContext updatedContext = agg.getContext().toBuilder().createNewAggregations(false).build();
            converted = new MetadataAggRel(agg.getCluster(), agg.getTraitSet(), converted, updatedContext);
        }
        call.transformTo(converted);
    } catch (Exception e) {
        logger.warn("Unable to use parquet metadata for ANALYZE: {}", e.getMessage(), e);
    }
}

Example 24 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by apache.

the class DbScanToIndexScanPrule method processWithIndexSelection.

private void processWithIndexSelection(IndexLogicalPlanCallContext indexContext, PlannerSettings settings, RexNode condition, IndexCollection collection, RexBuilder builder) {
    double totalRows = 0;
    double filterRows = totalRows;
    DrillScanRel scan = indexContext.scan;
    if (!(indexContext.scan.getGroupScan() instanceof DbGroupScan)) {
        return;
    }
    IndexConditionInfo.Builder infoBuilder = IndexConditionInfo.newBuilder(condition, collection, builder, indexContext.scan);
    IndexConditionInfo cInfo = infoBuilder.getCollectiveInfo(indexContext);
    boolean isValidIndexHint = infoBuilder.isValidIndexHint(indexContext);
    if (!cInfo.hasIndexCol) {
        logger.info("index_plan_info: No index columns are projected from the scan..continue.");
        return;
    }
    if (cInfo.indexCondition == null) {
        logger.info("index_plan_info: No conditions were found eligible for applying index lookup.");
        return;
    }
    if (!indexContext.indexHint.equals("") && !isValidIndexHint) {
        logger.warn("index_plan_info: Index Hint {} is not useful as index with that name is not available", indexContext.indexHint);
    }
    RexNode indexCondition = cInfo.indexCondition;
    RexNode remainderCondition = cInfo.remainderCondition;
    if (remainderCondition.isAlwaysTrue()) {
        remainderCondition = null;
    }
    logger.debug("index_plan_info: condition split into indexcondition: {} and remaindercondition: {}", indexCondition, remainderCondition);
    IndexableExprMarker indexableExprMarker = new IndexableExprMarker(indexContext.scan);
    indexCondition.accept(indexableExprMarker);
    indexContext.origMarker = indexableExprMarker;
    if (scan.getGroupScan() instanceof DbGroupScan) {
        // Initialize statistics
        DbGroupScan dbScan = ((DbGroupScan) scan.getGroupScan());
        if (settings.isStatisticsEnabled()) {
            dbScan.getStatistics().initialize(condition, scan, indexContext);
        }
        totalRows = dbScan.getRowCount(null, scan);
        filterRows = dbScan.getRowCount(condition, scan);
        double sel = filterRows / totalRows;
        if (totalRows != Statistics.ROWCOUNT_UNKNOWN && filterRows != Statistics.ROWCOUNT_UNKNOWN && !settings.isDisableFullTableScan() && !isValidIndexHint && sel > Math.max(settings.getIndexCoveringSelThreshold(), settings.getIndexNonCoveringSelThreshold())) {
            // If full table scan is not disabled, generate full table scan only plans if selectivity
            // is greater than covering and non-covering selectivity thresholds
            logger.info("index_plan_info: Skip index planning because filter selectivity: {} is greater than thresholds {}, {}", sel, settings.getIndexCoveringSelThreshold(), settings.getIndexNonCoveringSelThreshold());
            return;
        }
    }
    if (totalRows == Statistics.ROWCOUNT_UNKNOWN || totalRows == 0 || filterRows == Statistics.ROWCOUNT_UNKNOWN) {
        logger.warn("index_plan_info: Total row count is UNKNOWN or 0, or filterRows UNKNOWN; skip index planning");
        return;
    }
    List<IndexGroup> coveringIndexes = Lists.newArrayList();
    List<IndexGroup> nonCoveringIndexes = Lists.newArrayList();
    List<IndexGroup> intersectIndexes = Lists.newArrayList();
    // update sort expressions in context, it is needed for computing collation, so do it before IndexSelector
    IndexPlanUtils.updateSortExpression(indexContext, indexContext.sort != null ? indexContext.sort.collation.getFieldCollations() : null);
    IndexSelector selector = new IndexSelector(indexCondition, remainderCondition, indexContext, collection, builder, totalRows);
    for (IndexDescriptor indexDesc : collection) {
        logger.info("index_plan_info indexDescriptor: {}", indexDesc.toString());
        // check if any of the indexed fields of the index are present in the filter condition
        if (IndexPlanUtils.conditionIndexed(indexableExprMarker, indexDesc) != IndexPlanUtils.ConditionIndexed.NONE) {
            if (isValidIndexHint && !indexContext.indexHint.equals(indexDesc.getIndexName())) {
                logger.info("index_plan_info: Index {} is being discarded due to index Hint", indexDesc.getIndexName());
                continue;
            }
            FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
            selector.addIndex(indexDesc, IndexPlanUtils.isCoveringIndex(indexContext, functionInfo), indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : scan.getRowType().getFieldCount());
        }
    }
    // get the candidate indexes based on selection
    selector.getCandidateIndexes(infoBuilder, coveringIndexes, nonCoveringIndexes, intersectIndexes);
    if (logger.isDebugEnabled()) {
        StringBuilder strb = new StringBuilder();
        if (coveringIndexes.size() > 0) {
            strb.append("Covering indexes:");
            for (IndexGroup index : coveringIndexes) {
                strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
            }
        }
        if (nonCoveringIndexes.size() > 0) {
            strb.append("Non-covering indexes:");
            for (IndexGroup index : nonCoveringIndexes) {
                strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
            }
        }
        logger.debug("index_plan_info: IndexSelector return: {}", strb.toString());
    }
    GroupScan primaryTableScan = indexContext.scan.getGroupScan();
    // TODO: this logic for intersect should eventually be migrated to the IndexSelector
    if (coveringIndexes.size() == 0 && nonCoveringIndexes.size() > 1) {
        List<IndexDescriptor> indexList = Lists.newArrayList();
        for (IndexGroup index : nonCoveringIndexes) {
            IndexDescriptor indexDesc = index.getIndexProps().get(0).getIndexDesc();
            IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
            // Copy primary table statistics to index table
            idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
            indexList.add(index.getIndexProps().get(0).getIndexDesc());
        }
        Map<IndexDescriptor, IndexConditionInfo> indexInfoMap = infoBuilder.getIndexConditionMap(indexList);
        // no usable index
        if (indexInfoMap == null || indexInfoMap.size() == 0) {
            logger.info("index_plan_info: skipping intersect plan generation as there is no usable index");
            return;
        }
        // some part of filter condition needs to apply on primary table.
        if (indexInfoMap.size() > 1) {
            logger.info("index_plan_info: intersect plan is generated");
            if (logger.isDebugEnabled()) {
                List<String> indices = new ArrayList<>(nonCoveringIndexes.size());
                for (IndexGroup index : nonCoveringIndexes) {
                    indices.add(index.getIndexProps().get(0).getIndexDesc().getIndexName());
                }
                logger.debug("index_plan_info: intersect plan is generated on index list {}", indices);
            }
            boolean intersectPlanGenerated = false;
            // TODO: make sure the smallest selectivity of these indexes times rowcount smaller than broadcast threshold
            for (IndexGroup index : intersectIndexes) {
                List<IndexDescriptor> candidateDesc = Lists.newArrayList();
                for (IndexProperties candProp : index.getIndexProps()) {
                    candidateDesc.add(candProp.getIndexDesc());
                }
                Map<IndexDescriptor, IndexConditionInfo> intersectIdxInfoMap = infoBuilder.getIndexConditionMap(candidateDesc);
                IndexIntersectPlanGenerator planGen = new IndexIntersectPlanGenerator(indexContext, intersectIdxInfoMap, builder, settings);
                try {
                    planGen.go();
                    intersectPlanGenerated = true;
                } catch (Exception e) {
                    // If error while generating intersect plans, continue onto generating non-covering plans
                    logger.warn("index_plan_info: Exception while trying to generate intersect index plan", e);
                }
            }
            // If intersect plans are forced do not generate further non-covering plans
            if (intersectPlanGenerated && settings.isIndexIntersectPlanPreferred()) {
                return;
            }
        }
    }
    try {
        for (IndexGroup index : coveringIndexes) {
            IndexProperties indexProps = index.getIndexProps().get(0);
            IndexDescriptor indexDesc = indexProps.getIndexDesc();
            IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
            FunctionalIndexInfo indexInfo = indexDesc.getFunctionalInfo();
            indexCondition = indexProps.getLeadingColumnsFilter();
            remainderCondition = indexProps.getTotalRemainderFilter();
            // Copy primary table statistics to index table
            idxScan.setStatistics(((DbGroupScan) scan.getGroupScan()).getStatistics());
            logger.info("index_plan_info: Generating covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
            CoveringIndexPlanGenerator planGen = new CoveringIndexPlanGenerator(indexContext, indexInfo, idxScan, indexCondition, remainderCondition, builder, settings);
            planGen.go();
        }
    } catch (Exception e) {
        logger.warn("Exception while trying to generate covering index plan", e);
    }
    // First, check if the primary table scan supports creating a restricted scan
    if (primaryTableScan instanceof DbGroupScan && (((DbGroupScan) primaryTableScan).supportsRestrictedScan())) {
        try {
            for (IndexGroup index : nonCoveringIndexes) {
                IndexProperties indexProps = index.getIndexProps().get(0);
                IndexDescriptor indexDesc = indexProps.getIndexDesc();
                IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
                indexCondition = indexProps.getLeadingColumnsFilter();
                remainderCondition = indexProps.getTotalRemainderFilter();
                // Copy primary table statistics to index table
                idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
                logger.info("index_plan_info: Generating non-covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
                NonCoveringIndexPlanGenerator planGen = new NonCoveringIndexPlanGenerator(indexContext, indexDesc, idxScan, indexCondition, remainderCondition, builder, settings);
                planGen.go();
            }
        } catch (Exception e) {
            logger.warn("Exception while trying to generate non-covering index access plan", e);
        }
    }
}

Example 25 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by apache.

the class DrillPushLimitToScanRule method doOnMatch.

protected void doOnMatch(RelOptRuleCall call, DrillLimitRel limitRel, DrillScanRel scanRel, DrillProjectRel projectRel) {
    try {
        final int rowCountRequested = (int) limitRel.estimateRowCount(limitRel.getCluster().getMetadataQuery());
        final GroupScan newGroupScan = scanRel.getGroupScan().applyLimit(rowCountRequested);
        if (newGroupScan == null) {
            return;
        }
        DrillScanRel newScanRel = new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet(), scanRel.getTable(), newGroupScan, scanRel.getRowType(), scanRel.getColumns(), scanRel.partitionFilterPushdown());
        final RelNode newLimit;
        if (projectRel != null) {
            final RelNode newProject = projectRel.copy(projectRel.getTraitSet(), ImmutableList.of(newScanRel));
            newLimit = limitRel.copy(limitRel.getTraitSet(), ImmutableList.of(newProject));
        } else {
            newLimit = limitRel.copy(limitRel.getTraitSet(), ImmutableList.of(newScanRel));
        }
        call.transformTo(newLimit);
        logger.debug("Converted to a new DrillScanRel" + newScanRel.getGroupScan());
    } catch (Exception e) {
        logger.warn("Exception while using the pruned partitions.", e);
    }
}