Search in sources :

Example 96 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by apache.

the class DefaultSqlHandler method convertToPrel.

protected Prel convertToPrel(RelNode drel) throws RelConversionException, SqlUnsupportedException {
    Preconditions.checkArgument(drel.getConvention() == DrillRel.DRILL_LOGICAL);
    final RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
    Prel phyRelNode;
    try {
        final Stopwatch watch = Stopwatch.createStarted();
        final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits, false);
        phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
        // log externally as we need to finalize before traversing the tree.
        log(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, phyRelNode, logger, watch);
    } catch (RelOptPlanner.CannotPlanException ex) {
        logger.error(ex.getMessage());
        if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
            throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
        } else {
            throw ex;
        }
    }
    OptionManager queryOptions = context.getOptions();
    if (context.getPlannerSettings().isMemoryEstimationEnabled() && !MemoryEstimationVisitor.enoughMemory(phyRelNode, queryOptions, context.getActiveEndpoints().size())) {
        log("Not enough memory for this plan", phyRelNode, logger, null);
        logger.debug("Re-planning without hash operations.");
        queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false));
        queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false));
        try {
            final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits);
            phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
        } catch (RelOptPlanner.CannotPlanException ex) {
            logger.error(ex.getMessage());
            if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
                throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
            } else {
                throw ex;
            }
        }
    }
    /* The order of the following transformations is important */
    /*
     * 0.) For select * from join query, we need insert project on top of scan and a top project just
     * under screen operator. The project on top of scan will rename from * to T1*, while the top project
     * will rename T1* to *, before it output the final result. Only the top project will allow
     * duplicate columns, since user could "explicitly" ask for duplicate columns ( select *, col, *).
     * The rest of projects will remove the duplicate column when we generate POP in json format.
     */
    phyRelNode = StarColumnConverter.insertRenameProject(phyRelNode);
    /*
     * 1.)
     * Join might cause naming conflicts from its left and right child.
     * In such case, we have to insert Project to rename the conflicting names.
     */
    phyRelNode = JoinPrelRenameVisitor.insertRenameProject(phyRelNode);
    /*
     * 1.1) Swap left / right for INNER hash join, if left's row count is < (1 + margin) right's row count.
     * We want to have smaller dataset on the right side, since hash table builds on right side.
     */
    if (context.getPlannerSettings().isHashJoinSwapEnabled()) {
        phyRelNode = SwapHashJoinVisitor.swapHashJoin(phyRelNode, new Double(context.getPlannerSettings().getHashJoinSwapMarginFactor()));
    }
    if (context.getPlannerSettings().isParquetRowGroupFilterPushdownPlanningEnabled()) {
        phyRelNode = (Prel) transform(PlannerType.HEP_BOTTOM_UP, PlannerPhase.PHYSICAL_PARTITION_PRUNING, phyRelNode);
    }
    /*
     * 1.2) Break up all expressions with complex outputs into their own project operations
     */
    phyRelNode = phyRelNode.accept(new SplitUpComplexExpressions(config.getConverter().getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null);
    /*
     * 1.3) Projections that contain reference to flatten are rewritten as Flatten operators followed by Project
     */
    phyRelNode = phyRelNode.accept(new RewriteProjectToFlatten(config.getConverter().getTypeFactory(), context.getDrillOperatorTable()), null);
    /*
     * 2.)
     * Since our operators work via names rather than indices, we have to make to reorder any
     * output before we return data to the user as we may have accidentally shuffled things.
     * This adds a trivial project to reorder columns prior to output.
     */
    phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode);
    /*
     * 3.)
     * If two fragments are both estimated to be parallelization one, remove the exchange
     * separating them
     */
    phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);
    /* 5.)
     * if the client does not support complex types (Map, Repeated)
     * insert a project which which would convert
     */
    if (!context.getSession().isSupportComplexTypes()) {
        logger.debug("Client does not support complex types, add ComplexToJson operator.");
        phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
    }
    /* 6.)
     * Insert LocalExchange (mux and/or demux) nodes
     */
    phyRelNode = InsertLocalExchangeVisitor.insertLocalExchanges(phyRelNode, queryOptions);
    /* 7.)
     * Next, we add any required selection vector removers given the supported encodings of each
     * operator. This will ultimately move to a new trait but we're managing here for now to avoid
     * introducing new issues in planning before the next release
     */
    phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
    /* 8.)
     * Finally, Make sure that the no rels are repeats.
     * This could happen in the case of querying the same table twice as Optiq may canonicalize these.
     */
    phyRelNode = RelUniqifier.uniqifyGraph(phyRelNode);
    return phyRelNode;
}
Also used : UnsupportedRelOperatorException(org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException) RewriteProjectToFlatten(org.apache.drill.exec.planner.physical.visitor.RewriteProjectToFlatten) Stopwatch(com.google.common.base.Stopwatch) ArrayList(java.util.ArrayList) RelTraitSet(org.apache.calcite.plan.RelTraitSet) SplitUpComplexExpressions(org.apache.drill.exec.planner.physical.visitor.SplitUpComplexExpressions) RelOptPlanner(org.apache.calcite.plan.RelOptPlanner) OptionManager(org.apache.drill.exec.server.options.OptionManager) Prel(org.apache.drill.exec.planner.physical.Prel) RelNode(org.apache.calcite.rel.RelNode)

Example 97 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class JoinPruleBase method createBroadcastPlan.

// Create join plan with left child ANY distributed and right child BROADCAST distributed. If the physical join type
// is MergeJoin, a collation must be provided for both left and right child and the plan will contain sort converter
// if necessary to provide the collation.
protected void createBroadcastPlan(final RelOptRuleCall call, final DrillJoinRel join, final RexNode joinCondition, final PhysicalJoinType physicalJoinType, final RelNode left, final RelNode right, final RelCollation collationLeft, final RelCollation collationRight) throws InvalidRelException {
    DrillDistributionTrait distBroadcastRight = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.BROADCAST_DISTRIBUTED);
    RelTraitSet traitsRight = null;
    RelTraitSet traitsLeft = left.getTraitSet().plus(Prel.DRILL_PHYSICAL);
    if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
        assert collationLeft != null && collationRight != null;
        traitsLeft = traitsLeft.plus(collationLeft);
        traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collationRight).plus(distBroadcastRight);
    } else if (physicalJoinType == PhysicalJoinType.HASH_JOIN || physicalJoinType == PhysicalJoinType.NESTEDLOOP_JOIN) {
        traitsRight = right.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distBroadcastRight);
    }
    final RelNode convertedLeft = convert(left, traitsLeft);
    final RelNode convertedRight = convert(right, traitsRight);
    boolean traitProp = false;
    if (traitProp) {
        if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
            new SubsetTransformer<DrillJoinRel, InvalidRelException>(call) {

                @Override
                public RelNode convertChild(final DrillJoinRel join, final RelNode rel) throws InvalidRelException {
                    DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
                    RelTraitSet newTraitsLeft = newTraitSet(Prel.DRILL_PHYSICAL, collationLeft, toDist);
                    RelNode newLeft = convert(left, newTraitsLeft);
                    return new MergeJoinPrel(join.getCluster(), newTraitsLeft, newLeft, convertedRight, joinCondition, join.getJoinType());
                }
            }.go(join, convertedLeft);
        } else if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
            new SubsetTransformer<DrillJoinRel, InvalidRelException>(call) {

                @Override
                public RelNode convertChild(final DrillJoinRel join, final RelNode rel) throws InvalidRelException {
                    DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
                    RelTraitSet newTraitsLeft = newTraitSet(Prel.DRILL_PHYSICAL, toDist);
                    RelNode newLeft = convert(left, newTraitsLeft);
                    return new HashJoinPrel(join.getCluster(), newTraitsLeft, newLeft, convertedRight, joinCondition, join.getJoinType());
                }
            }.go(join, convertedLeft);
        } else if (physicalJoinType == PhysicalJoinType.NESTEDLOOP_JOIN) {
            new SubsetTransformer<DrillJoinRel, InvalidRelException>(call) {

                @Override
                public RelNode convertChild(final DrillJoinRel join, final RelNode rel) throws InvalidRelException {
                    DrillDistributionTrait toDist = rel.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
                    RelTraitSet newTraitsLeft = newTraitSet(Prel.DRILL_PHYSICAL, toDist);
                    RelNode newLeft = convert(left, newTraitsLeft);
                    return new NestedLoopJoinPrel(join.getCluster(), newTraitsLeft, newLeft, convertedRight, joinCondition, join.getJoinType());
                }
            }.go(join, convertedLeft);
        }
    } else {
        if (physicalJoinType == PhysicalJoinType.MERGE_JOIN) {
            call.transformTo(new MergeJoinPrel(join.getCluster(), convertedLeft.getTraitSet(), convertedLeft, convertedRight, joinCondition, join.getJoinType()));
        } else if (physicalJoinType == PhysicalJoinType.HASH_JOIN) {
            final RelTraitSet traitSet = PrelUtil.removeCollation(convertedLeft.getTraitSet(), call);
            call.transformTo(new HashJoinPrel(join.getCluster(), traitSet, convertedLeft, convertedRight, joinCondition, join.getJoinType()));
        } else if (physicalJoinType == PhysicalJoinType.NESTEDLOOP_JOIN) {
            call.transformTo(new NestedLoopJoinPrel(join.getCluster(), convertedLeft.getTraitSet(), convertedLeft, convertedRight, joinCondition, join.getJoinType()));
        }
    }
}
Also used : InvalidRelException(org.apache.calcite.rel.InvalidRelException) DrillJoinRel(org.apache.drill.exec.planner.logical.DrillJoinRel) RelNode(org.apache.calcite.rel.RelNode) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Example 98 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class ProjectPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillProjectRel project = (DrillProjectRel) call.rel(0);
    final RelNode input = project.getInput();
    RelTraitSet traits = input.getTraitSet().plus(Prel.DRILL_PHYSICAL);
    RelNode convertedInput = convert(input, traits);
    // Maintain two different map for distribution trait and collation trait.
    // For now, the only difference comes from the way how cast function impacts propagating trait.
    final Map<Integer, Integer> distributionMap = getDistributionMap(project);
    final Map<Integer, Integer> collationMap = getCollationMap(project);
    boolean traitPull = new ProjectTraitPull(call, distributionMap, collationMap).go(project, convertedInput);
    if (!traitPull) {
        call.transformTo(new ProjectPrel(project.getCluster(), convertedInput.getTraitSet(), convertedInput, project.getProjects(), project.getRowType()));
    }
}
Also used : RelNode(org.apache.calcite.rel.RelNode) DrillProjectRel(org.apache.drill.exec.planner.logical.DrillProjectRel) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Example 99 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class FilterPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillFilterRel filter = (DrillFilterRel) call.rel(0);
    final RelNode input = filter.getInput();
    RelTraitSet traits = input.getTraitSet().plus(Prel.DRILL_PHYSICAL);
    RelNode convertedInput = convert(input, traits);
    boolean transform = new Subset(call).go(filter, convertedInput);
    if (!transform) {
        call.transformTo(new FilterPrel(filter.getCluster(), convertedInput.getTraitSet(), convertedInput, filter.getCondition()));
    }
}
Also used : RelNode(org.apache.calcite.rel.RelNode) DrillFilterRel(org.apache.drill.exec.planner.logical.DrillFilterRel) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Example 100 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class HashAggPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    if (!PrelUtil.getPlannerSettings(call.getPlanner()).isHashAggEnabled()) {
        return;
    }
    final DrillAggregateRel aggregate = call.rel(0);
    final RelNode input = call.rel(1);
    if (aggregate.containsDistinctCall() || aggregate.getGroupCount() == 0) {
        // if there are no grouping keys
        return;
    }
    RelTraitSet traits;
    try {
        if (aggregate.getGroupSet().isEmpty()) {
            DrillDistributionTrait singleDist = DrillDistributionTrait.SINGLETON;
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(singleDist);
            createTransformRequest(call, aggregate, input, traits);
        } else {
            // hash distribute on all grouping keys
            DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, true)));
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distOnAllKeys);
            createTransformRequest(call, aggregate, input, traits);
            // hash distribute on single grouping key
            DrillDistributionTrait distOnOneKey = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionField(aggregate, false)));
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(distOnOneKey);
            createTransformRequest(call, aggregate, input, traits);
            if (create2PhasePlan(call, aggregate)) {
                traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
                RelNode convertedInput = convert(input, traits);
                new TwoPhaseSubset(call, distOnAllKeys).go(aggregate, convertedInput);
            }
        }
    } catch (InvalidRelException e) {
        tracer.warn(e.toString());
    }
}
Also used : InvalidRelException(org.apache.calcite.rel.InvalidRelException) RelNode(org.apache.calcite.rel.RelNode) DrillAggregateRel(org.apache.drill.exec.planner.logical.DrillAggregateRel) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Aggregations

RelTraitSet (org.apache.calcite.plan.RelTraitSet)190 RelNode (org.apache.calcite.rel.RelNode)111 RelOptCluster (org.apache.calcite.plan.RelOptCluster)38 RelCollation (org.apache.calcite.rel.RelCollation)35 RelOptPlanner (org.apache.calcite.plan.RelOptPlanner)26 RexNode (org.apache.calcite.rex.RexNode)24 ArrayList (java.util.ArrayList)20 InvalidRelException (org.apache.calcite.rel.InvalidRelException)19 RelDataType (org.apache.calcite.rel.type.RelDataType)14 SqlNode (org.apache.calcite.sql.SqlNode)14 List (java.util.List)13 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)13 RelOptTable (org.apache.calcite.plan.RelOptTable)11 Sort (org.apache.calcite.rel.core.Sort)11 RelMetadataQuery (org.apache.calcite.rel.metadata.RelMetadataQuery)11 ImmutableList (com.google.common.collect.ImmutableList)10 RelFieldCollation (org.apache.calcite.rel.RelFieldCollation)10 Test (org.junit.Test)9 Table (org.apache.calcite.schema.Table)8 RexInputRef (org.apache.calcite.rex.RexInputRef)7