Examples with GroupScan org.apache.drill.exec.physical.base.GroupScan used on opensource projects

Example 16 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by axbaretto.

the class ConvertCountToDirectScan method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillAggregateRel agg = (DrillAggregateRel) call.rel(0);
    final DrillScanRel scan = (DrillScanRel) call.rel(call.rels.length - 1);
    final DrillProjectRel project = call.rels.length == 3 ? (DrillProjectRel) call.rel(1) : null;
    final GroupScan oldGrpScan = scan.getGroupScan();
    final PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
    // 3) No distinct agg call.
    if (!(oldGrpScan.getScanStats(settings).getGroupScanProperty().hasExactRowCount() && agg.getGroupCount() == 0 && !agg.containsDistinctCall())) {
        return;
    }
    Map<String, Long> result = collectCounts(settings, agg, scan, project);
    logger.trace("Calculated the following aggregate counts: ", result);
    // if could not determine the counts, rule won't be applied
    if (result.isEmpty()) {
        return;
    }
    final RelDataType scanRowType = constructDataType(agg, result.keySet());
    final DynamicPojoRecordReader<Long> reader = new DynamicPojoRecordReader<>(buildSchema(scanRowType.getFieldNames()), Collections.singletonList((List<Long>) new ArrayList<>(result.values())));
    final ScanStats scanStats = new ScanStats(ScanStats.GroupScanProperty.EXACT_ROW_COUNT, 1, 1, scanRowType.getFieldCount());
    final GroupScan directScan = new MetadataDirectGroupScan(reader, oldGrpScan.getFiles(), scanStats);
    final ScanPrel newScan = ScanPrel.create(scan, scan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON), directScan, scanRowType);
    final ProjectPrel newProject = new ProjectPrel(agg.getCluster(), agg.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON), newScan, prepareFieldExpressions(scanRowType), agg.getRowType());
    call.transformTo(newProject);
}

Example 17 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by axbaretto.

the class DrillPushLimitToScanRule method doOnMatch.

protected void doOnMatch(RelOptRuleCall call, DrillLimitRel limitRel, DrillScanRel scanRel, DrillProjectRel projectRel) {
    try {
        final int rowCountRequested = (int) limitRel.getRows();
        final GroupScan newGroupScan = scanRel.getGroupScan().applyLimit(rowCountRequested);
        if (newGroupScan == null) {
            return;
        }
        DrillScanRel newScanRel = new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet(), scanRel.getTable(), newGroupScan, scanRel.getRowType(), scanRel.getColumns(), scanRel.partitionFilterPushdown());
        final RelNode newLimit;
        if (projectRel != null) {
            final RelNode newProject = projectRel.copy(projectRel.getTraitSet(), ImmutableList.of((RelNode) newScanRel));
            newLimit = limitRel.copy(limitRel.getTraitSet(), ImmutableList.of((RelNode) newProject));
        } else {
            newLimit = limitRel.copy(limitRel.getTraitSet(), ImmutableList.of((RelNode) newScanRel));
        }
        call.transformTo(newLimit);
        logger.debug("Converted to a new DrillScanRel" + newScanRel.getGroupScan());
    } catch (Exception e) {
        logger.warn("Exception while using the pruned partitions.", e);
    }
}

Example 18 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by axbaretto.

the class ParquetPartitionDescriptor method createTableScan.

@Override
public TableScan createTableScan(List<PartitionLocation> newPartitionLocation, String cacheFileRoot, boolean wasAllPartitionsPruned, MetadataContext metaContext) throws Exception {
    List<String> newFiles = Lists.newArrayList();
    for (final PartitionLocation location : newPartitionLocation) {
        newFiles.add(location.getEntirePartitionLocation());
    }
    final GroupScan newGroupScan = createNewGroupScan(newFiles, cacheFileRoot, wasAllPartitionsPruned, metaContext);
    return new DrillScanRel(scanRel.getCluster(), scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), scanRel.getTable(), newGroupScan, scanRel.getRowType(), scanRel.getColumns(), true);
}

Example 19 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by axbaretto.

the class ScanPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillScanRel scan = (DrillScanRel) call.rel(0);
    GroupScan groupScan = scan.getGroupScan();
    DrillDistributionTrait partition = (groupScan.getMaxParallelizationWidth() > 1 || groupScan.getDistributionAffinity() == DistributionAffinity.HARD) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
    final RelTraitSet traits = scan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition);
    final DrillScanPrel newScan = ScanPrel.create(scan, traits, groupScan, scan.getRowType());
    call.transformTo(newScan);
}

Example 20 with GroupScan

use of org.apache.drill.exec.physical.base.GroupScan in project drill by axbaretto.

the class ParquetPushDownFilter method doOnMatch.

protected void doOnMatch(RelOptRuleCall call, FilterPrel filter, ProjectPrel project, ScanPrel scan) {
    ParquetGroupScan groupScan = (ParquetGroupScan) scan.getGroupScan();
    if (groupScan.getFilter() != null && !groupScan.getFilter().equals(ValueExpressions.BooleanExpression.TRUE)) {
        return;
    }
    RexNode condition = null;
    if (project == null) {
        condition = filter.getCondition();
    } else {
        // get the filter as if it were below the projection.
        condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project);
    }
    if (condition == null || condition.equals(ValueExpressions.BooleanExpression.TRUE)) {
        return;
    }
    // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
    // then we could not pushed down. Otherwise, it's qualified to be pushed down.
    final List<RexNode> predList = RelOptUtil.conjunctions(condition);
    final List<RexNode> qualifiedPredList = Lists.newArrayList();
    for (final RexNode pred : predList) {
        if (DrillRelOptUtil.findOperators(pred, ImmutableList.of(), BANNED_OPERATORS) == null) {
            qualifiedPredList.add(pred);
        }
    }
    final RexNode qualifedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
    if (qualifedPred == null) {
        return;
    }
    LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifedPred);
    Stopwatch timer = Stopwatch.createStarted();
    final GroupScan newGroupScan = groupScan.applyFilter(conditionExp, optimizerContext, optimizerContext.getFunctionRegistry(), optimizerContext.getPlannerSettings().getOptions());
    logger.info("Took {} ms to apply filter on parquet row groups. ", timer.elapsed(TimeUnit.MILLISECONDS));
    if (newGroupScan == null) {
        return;
    }
    final ScanPrel newScanRel = ScanPrel.create(scan, scan.getTraitSet(), newGroupScan, scan.getRowType());
    RelNode inputRel = newScanRel;
    if (project != null) {
        inputRel = project.copy(project.getTraitSet(), ImmutableList.of(inputRel));
    }
    final RelNode newFilter = filter.copy(filter.getTraitSet(), ImmutableList.<RelNode>of(inputRel));
    call.transformTo(newFilter);
}