Examples with RelOptCost org.apache.calcite.plan.RelOptCost used on opensource projects

Example 1 with RelOptCost

use of org.apache.calcite.plan.RelOptCost in project calcite by apache.

the class Correlate method computeSelfCost.

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
    double rowCount = mq.getRowCount(this);
    final double rightRowCount = right.estimateRowCount(mq);
    final double leftRowCount = left.estimateRowCount(mq);
    if (Double.isInfinite(leftRowCount) || Double.isInfinite(rightRowCount)) {
        return planner.getCostFactory().makeInfiniteCost();
    }
    Double restartCount = mq.getRowCount(getLeft());
    // RelMetadataQuery.getCumulativeCost(getRight()); does not work for
    // RelSubset, so we ask planner to cost-estimate right relation
    RelOptCost rightCost = planner.getCost(getRight(), mq);
    RelOptCost rescanCost = rightCost.multiplyBy(Math.max(1.0, restartCount - 1));
    return planner.getCostFactory().makeCost(rowCount + /* generate results */
    leftRowCount, /* scan left results */
    0, 0).plus(rescanCost);
}

Example 2 with RelOptCost

use of org.apache.calcite.plan.RelOptCost in project calcite by apache.

the class RelSubset method computeBestCost.

// ~ Methods ----------------------------------------------------------------
/**
 * Computes the best {@link RelNode} in this subset.
 *
 * <p>Only necessary when a subset is created in a set that has subsets that
 * subsume it. Rationale:</p>
 *
 * <ol>
 * <li>If the are no subsuming subsets, the subset is initially empty.</li>
 * <li>After creation, {@code best} and {@code bestCost} are maintained
 *    incrementally by {@link #propagateCostImprovements0} and
 *    {@link RelSet#mergeWith(VolcanoPlanner, RelSet)}.</li>
 * </ol>
 */
private void computeBestCost(RelOptPlanner planner) {
    bestCost = planner.getCostFactory().makeInfiniteCost();
    final RelMetadataQuery mq = getCluster().getMetadataQuery();
    for (RelNode rel : getRels()) {
        final RelOptCost cost = planner.getCost(rel, mq);
        if (cost.isLt(bestCost)) {
            bestCost = cost;
            best = rel;
        }
    }
}

Example 3 with RelOptCost

use of org.apache.calcite.plan.RelOptCost in project hive by apache.

the class HiveAggregateJoinTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    try {
        final Aggregate aggregate = call.rel(0);
        final Join join = call.rel(1);
        final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
        final RelBuilder relBuilder = call.builder();
        // If any aggregate call has a filter, bail out
        for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
            if (aggregateCall.getAggregation().unwrap(SqlSplittableAggFunction.class) == null) {
                return;
            }
            if (aggregateCall.filterArg >= 0) {
                return;
            }
        }
        // aggregate operator
        if (join.getJoinType() != JoinRelType.INNER) {
            return;
        }
        if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) {
            return;
        }
        boolean groupingUnique = isGroupingUnique(join, aggregate.getGroupSet());
        if (!groupingUnique && !costBased) {
            // there is no need to check further - the transformation may not happen
            return;
        }
        // Do the columns used by the join appear in the output of the aggregate?
        final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
        final RelMetadataQuery mq = call.getMetadataQuery();
        final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates);
        final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition());
        final boolean allColumnsInAggregate = keyColumns.contains(joinColumns);
        final ImmutableBitSet belowAggregateColumns = aggregateColumns.union(joinColumns);
        // Split join condition
        final List<Integer> leftKeys = Lists.newArrayList();
        final List<Integer> rightKeys = Lists.newArrayList();
        final List<Boolean> filterNulls = Lists.newArrayList();
        RexNode nonEquiConj = RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(), join.getCondition(), leftKeys, rightKeys, filterNulls);
        // If it contains non-equi join conditions, we bail out
        if (!nonEquiConj.isAlwaysTrue()) {
            return;
        }
        // Push each aggregate function down to each side that contains all of its
        // arguments. Note that COUNT(*), because it has no arguments, can go to
        // both sides.
        final Map<Integer, Integer> map = new HashMap<>();
        final List<Side> sides = new ArrayList<>();
        int uniqueCount = 0;
        int offset = 0;
        int belowOffset = 0;
        for (int s = 0; s < 2; s++) {
            final Side side = new Side();
            final RelNode joinInput = join.getInput(s);
            int fieldCount = joinInput.getRowType().getFieldCount();
            final ImmutableBitSet fieldSet = ImmutableBitSet.range(offset, offset + fieldCount);
            final ImmutableBitSet belowAggregateKeyNotShifted = belowAggregateColumns.intersect(fieldSet);
            for (Ord<Integer> c : Ord.zip(belowAggregateKeyNotShifted)) {
                map.put(c.e, belowOffset + c.i);
            }
            final ImmutableBitSet belowAggregateKey = belowAggregateKeyNotShifted.shift(-offset);
            final boolean unique;
            if (!allowFunctions) {
                assert aggregate.getAggCallList().isEmpty();
                // If there are no functions, it doesn't matter as much whether we
                // aggregate the inputs before the join, because there will not be
                // any functions experiencing a cartesian product effect.
                // 
                // But finding out whether the input is already unique requires a call
                // to areColumnsUnique that currently (until [CALCITE-1048] "Make
                // metadata more robust" is fixed) places a heavy load on
                // the metadata system.
                // 
                // So we choose to imagine the the input is already unique, which is
                // untrue but harmless.
                // 
                unique = true;
            } else {
                final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey, true);
                unique = unique0 != null && unique0;
            }
            if (unique) {
                ++uniqueCount;
                relBuilder.push(joinInput);
                relBuilder.project(belowAggregateKey.asList().stream().map(relBuilder::field).collect(Collectors.toList()));
                side.newInput = relBuilder.build();
            } else {
                List<AggregateCall> belowAggCalls = new ArrayList<>();
                final SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry = registry(belowAggCalls);
                final Mappings.TargetMapping mapping = s == 0 ? Mappings.createIdentity(fieldCount) : Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
                for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
                    final SqlAggFunction aggregation = aggCall.e.getAggregation();
                    final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
                    final AggregateCall call1;
                    if (fieldSet.contains(ImmutableBitSet.of(aggCall.e.getArgList()))) {
                        call1 = splitter.split(aggCall.e, mapping);
                    } else {
                        call1 = splitter.other(rexBuilder.getTypeFactory(), aggCall.e);
                    }
                    if (call1 != null) {
                        side.split.put(aggCall.i, belowAggregateKey.cardinality() + belowAggCallRegistry.register(call1));
                    }
                }
                side.newInput = relBuilder.push(joinInput).aggregate(relBuilder.groupKey(belowAggregateKey, null), belowAggCalls).build();
            }
            offset += fieldCount;
            belowOffset += side.newInput.getRowType().getFieldCount();
            sides.add(side);
        }
        if (uniqueCount == 2) {
            // invocation of this rule; if we continue we might loop forever.
            return;
        }
        // Update condition
        final Mapping mapping = (Mapping) Mappings.target(map::get, join.getRowType().getFieldCount(), belowOffset);
        final RexNode newCondition = RexUtil.apply(mapping, join.getCondition());
        // Create new join
        relBuilder.push(sides.get(0).newInput).push(sides.get(1).newInput).join(join.getJoinType(), newCondition);
        // Aggregate above to sum up the sub-totals
        final List<AggregateCall> newAggCalls = new ArrayList<>();
        final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
        final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
        final List<RexNode> projects = new ArrayList<>(rexBuilder.identityProjects(relBuilder.peek().getRowType()));
        for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
            final SqlAggFunction aggregation = aggCall.e.getAggregation();
            final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
            final Integer leftSubTotal = sides.get(0).split.get(aggCall.i);
            final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
            newAggCalls.add(splitter.topSplit(rexBuilder, registry(projects), groupIndicatorCount, relBuilder.peek().getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
        }
        relBuilder.project(projects);
        boolean aggConvertedToProjects = false;
        if (allColumnsInAggregate) {
            // let's see if we can convert aggregate into projects
            List<RexNode> projects2 = new ArrayList<>();
            for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
                projects2.add(relBuilder.field(key));
            }
            for (AggregateCall newAggCall : newAggCalls) {
                final SqlSplittableAggFunction splitter = newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
                if (splitter != null) {
                    final RelDataType rowType = relBuilder.peek().getRowType();
                    projects2.add(splitter.singleton(rexBuilder, rowType, newAggCall));
                }
            }
            if (projects2.size() == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
                // We successfully converted agg calls into projects.
                relBuilder.project(projects2);
                aggConvertedToProjects = true;
            }
        }
        if (!aggConvertedToProjects) {
            relBuilder.aggregate(relBuilder.groupKey(Mappings.apply(mapping, aggregate.getGroupSet()), Mappings.apply2(mapping, aggregate.getGroupSets())), newAggCalls);
        }
        RelNode r = relBuilder.build();
        boolean transform = false;
        if (uniqueBased && aggConvertedToProjects) {
            transform = groupingUnique;
        }
        if (!transform && costBased) {
            RelOptCost afterCost = mq.getCumulativeCost(r);
            RelOptCost beforeCost = mq.getCumulativeCost(aggregate);
            transform = afterCost.isLt(beforeCost);
        }
        if (transform) {
            call.transformTo(r);
        }
    } catch (Exception e) {
        if (noColsMissingStats.get() > 0) {
            LOG.warn("Missing column stats (see previous messages), skipping aggregate-join transpose in CBO");
            noColsMissingStats.set(0);
        } else {
            throw e;
        }
    }
}

Example 4 with RelOptCost

use of org.apache.calcite.plan.RelOptCost in project hive by apache.

the class HiveCostModel method getJoinCost.

public RelOptCost getJoinCost(HiveJoin join) {
    // Select algorithm with min cost
    JoinAlgorithm joinAlgorithm = null;
    RelOptCost minJoinCost = null;
    if (LOG.isTraceEnabled()) {
        LOG.trace("Join algorithm selection for:\n" + RelOptUtil.toString(join));
    }
    for (JoinAlgorithm possibleAlgorithm : this.joinAlgorithms) {
        if (!possibleAlgorithm.isExecutable(join)) {
            continue;
        }
        RelOptCost joinCost = possibleAlgorithm.getCost(join);
        LOG.trace("{} cost: {}", possibleAlgorithm, joinCost);
        if (minJoinCost == null || joinCost.isLt(minJoinCost)) {
            joinAlgorithm = possibleAlgorithm;
            minJoinCost = joinCost;
        }
    }
    LOG.trace("{} selected", joinAlgorithm);
    join.setJoinAlgorithm(joinAlgorithm);
    join.setJoinCost(minJoinCost);
    return minJoinCost;
}

Example 5 with RelOptCost

use of org.apache.calcite.plan.RelOptCost in project hive by apache.

the class HiveVolcanoPlanner method getCost.

/**
 * The method extends the logic of the super method to decrease
 * the cost of the plan if it contains materialized views
 * (heuristic).
 */
public RelOptCost getCost(RelNode rel, RelMetadataQuery mq) {
    assert rel != null : "pre-condition: rel != null";
    if (rel instanceof RelSubset) {
        // Get cost of the subset, best rel may have been chosen or not
        RelSubset subset = (RelSubset) rel;
        if (subset.getBest() != null) {
            return getCost(subset.getBest(), mq);
        }
        return costFactory.makeInfiniteCost();
    }
    if (rel.getTraitSet().getTrait(ConventionTraitDef.INSTANCE) == Convention.NONE) {
        return costFactory.makeInfiniteCost();
    }
    // We get the cost of the operator
    RelOptCost cost = mq.getNonCumulativeCost(rel);
    if (!costFactory.makeZeroCost().isLt(cost)) {
        // cost must be positive, so nudge it
        cost = costFactory.makeTinyCost();
    }
    // If this operator has a materialized view below,
    // we make its cost tiny and adjust the cost of its
    // inputs
    boolean usesMaterializedViews = false;
    Multimap<Class<? extends RelNode>, RelNode> nodeTypes = mq.getNodeTypes(rel);
    for (RelNode scan : nodeTypes.get(TableScan.class)) {
        if (scan.getTable() instanceof RelOptHiveTable) {
            RelOptHiveTable relOptHiveTable = (RelOptHiveTable) scan.getTable();
            if (relOptHiveTable.getHiveTableMD().isMaterializedView()) {
                usesMaterializedViews = true;
                break;
            }
        }
    }
    if (isHeuristic && usesMaterializedViews) {
        // If a child of this expression uses a materialized view,
        // then we decrease its cost by a certain factor. This is
        // useful for e.g. partial rewritings, where a part of plan
        // does not use the materialization, but we still want to
        // decrease its cost so it is chosen instead of the original
        // plan
        cost = cost.multiplyBy(RelOptUtil.EPSILON);
        if (!costFactory.makeZeroCost().isLt(cost)) {
            // cost must be positive, so nudge it
            cost = costFactory.makeTinyCost();
        }
        for (RelNode input : rel.getInputs()) {
            cost = cost.plus(getCost(input, mq));
        }
    } else {
        // No materialized view or not heuristic approach, normal costing
        for (RelNode input : rel.getInputs()) {
            cost = cost.plus(getCost(input, mq));
        }
    }
    return cost;
}