Search in sources :

Example 1 with ParquetGroupScan

use of org.apache.drill.exec.store.parquet.ParquetGroupScan in project drill by apache.

the class ParquetPartitionDescriptor method populatePartitionVectors.

@Override
public void populatePartitionVectors(ValueVector[] vectors, List<PartitionLocation> partitions, BitSet partitionColumnBitSet, Map<Integer, String> fieldNameMap) {
    int record = 0;
    for (PartitionLocation partitionLocation : partitions) {
        for (int partitionColumnIndex : BitSets.toIter(partitionColumnBitSet)) {
            SchemaPath column = SchemaPath.getSimplePath(fieldNameMap.get(partitionColumnIndex));
            ((ParquetGroupScan) scanRel.getGroupScan()).populatePruningVector(vectors[partitionColumnIndex], record, column, partitionLocation.getEntirePartitionLocation());
        }
        record++;
    }
    for (ValueVector v : vectors) {
        if (v == null) {
            continue;
        }
        v.getMutator().setValueCount(partitions.size());
    }
}
Also used : ValueVector(org.apache.drill.exec.vector.ValueVector) SchemaPath(org.apache.drill.common.expression.SchemaPath) ParquetGroupScan(org.apache.drill.exec.store.parquet.ParquetGroupScan)

Example 2 with ParquetGroupScan

use of org.apache.drill.exec.store.parquet.ParquetGroupScan in project drill by apache.

the class ParquetPruneScanRule method getFilterOnScanParquet.

public static final RelOptRule getFilterOnScanParquet(OptimizerRulesContext optimizerRulesContext) {
    return new PruneScanRule(RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)), "PruneScanRule:Filter_On_Scan_Parquet", optimizerRulesContext) {

        @Override
        public PartitionDescriptor getPartitionDescriptor(PlannerSettings settings, TableScan scanRel) {
            return new ParquetPartitionDescriptor(settings, (DrillScanRel) scanRel);
        }

        @Override
        public boolean matches(RelOptRuleCall call) {
            final DrillScanRel scan = call.rel(1);
            GroupScan groupScan = scan.getGroupScan();
            // this rule is applicable only for parquet based partition pruning
            if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) {
                return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown();
            } else {
                return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
            }
        }

        @Override
        public void onMatch(RelOptRuleCall call) {
            final DrillFilterRel filterRel = call.rel(0);
            final DrillScanRel scanRel = call.rel(1);
            doOnMatch(call, filterRel, null, scanRel);
        }
    };
}
Also used : FileGroupScan(org.apache.drill.exec.physical.base.FileGroupScan) GroupScan(org.apache.drill.exec.physical.base.GroupScan) ParquetGroupScan(org.apache.drill.exec.store.parquet.ParquetGroupScan) TableScan(org.apache.calcite.rel.core.TableScan) DrillScanRel(org.apache.drill.exec.planner.logical.DrillScanRel) PlannerSettings(org.apache.drill.exec.planner.physical.PlannerSettings) DrillFilterRel(org.apache.drill.exec.planner.logical.DrillFilterRel) ParquetGroupScan(org.apache.drill.exec.store.parquet.ParquetGroupScan) RelOptRuleCall(org.apache.calcite.plan.RelOptRuleCall) ParquetPartitionDescriptor(org.apache.drill.exec.planner.ParquetPartitionDescriptor)

Example 3 with ParquetGroupScan

use of org.apache.drill.exec.store.parquet.ParquetGroupScan in project drill by apache.

the class ParquetPruneScanRule method getFilterOnProjectParquet.

public static final RelOptRule getFilterOnProjectParquet(OptimizerRulesContext optimizerRulesContext) {
    return new PruneScanRule(RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), "PruneScanRule:Filter_On_Project_Parquet", optimizerRulesContext) {

        @Override
        public PartitionDescriptor getPartitionDescriptor(PlannerSettings settings, TableScan scanRel) {
            return new ParquetPartitionDescriptor(settings, (DrillScanRel) scanRel);
        }

        @Override
        public boolean matches(RelOptRuleCall call) {
            final DrillScanRel scan = call.rel(2);
            GroupScan groupScan = scan.getGroupScan();
            // this rule is applicable only for parquet based partition pruning
            if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) {
                return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown();
            } else {
                return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
            }
        }

        @Override
        public void onMatch(RelOptRuleCall call) {
            final DrillFilterRel filterRel = call.rel(0);
            final DrillProjectRel projectRel = call.rel(1);
            final DrillScanRel scanRel = call.rel(2);
            doOnMatch(call, filterRel, projectRel, scanRel);
        }
    };
}
Also used : FileGroupScan(org.apache.drill.exec.physical.base.FileGroupScan) GroupScan(org.apache.drill.exec.physical.base.GroupScan) ParquetGroupScan(org.apache.drill.exec.store.parquet.ParquetGroupScan) TableScan(org.apache.calcite.rel.core.TableScan) DrillScanRel(org.apache.drill.exec.planner.logical.DrillScanRel) PlannerSettings(org.apache.drill.exec.planner.physical.PlannerSettings) DrillProjectRel(org.apache.drill.exec.planner.logical.DrillProjectRel) DrillFilterRel(org.apache.drill.exec.planner.logical.DrillFilterRel) ParquetGroupScan(org.apache.drill.exec.store.parquet.ParquetGroupScan) RelOptRuleCall(org.apache.calcite.plan.RelOptRuleCall) ParquetPartitionDescriptor(org.apache.drill.exec.planner.ParquetPartitionDescriptor)

Aggregations

ParquetGroupScan (org.apache.drill.exec.store.parquet.ParquetGroupScan)3 RelOptRuleCall (org.apache.calcite.plan.RelOptRuleCall)2 TableScan (org.apache.calcite.rel.core.TableScan)2 FileGroupScan (org.apache.drill.exec.physical.base.FileGroupScan)2 GroupScan (org.apache.drill.exec.physical.base.GroupScan)2 ParquetPartitionDescriptor (org.apache.drill.exec.planner.ParquetPartitionDescriptor)2 DrillFilterRel (org.apache.drill.exec.planner.logical.DrillFilterRel)2 DrillScanRel (org.apache.drill.exec.planner.logical.DrillScanRel)2 PlannerSettings (org.apache.drill.exec.planner.physical.PlannerSettings)2 SchemaPath (org.apache.drill.common.expression.SchemaPath)1 DrillProjectRel (org.apache.drill.exec.planner.logical.DrillProjectRel)1 ValueVector (org.apache.drill.exec.vector.ValueVector)1