Search in sources :

Example 36 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveSortMergeRule method onMatch.

// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit topSortLimit = call.rel(0);
    final HiveSortLimit bottomSortLimit = call.rel(1);
    final RexNode newOffset;
    final RexNode newLimit;
    if (HiveCalciteUtil.limitRelNode(bottomSortLimit)) {
        final RexBuilder rexBuilder = topSortLimit.getCluster().getRexBuilder();
        int topOffset = topSortLimit.offset == null ? 0 : RexLiteral.intValue(topSortLimit.offset);
        int topLimit = RexLiteral.intValue(topSortLimit.fetch);
        int bottomOffset = bottomSortLimit.offset == null ? 0 : RexLiteral.intValue(bottomSortLimit.offset);
        int bottomLimit = RexLiteral.intValue(bottomSortLimit.fetch);
        // Three different cases
        if (topOffset + topLimit <= bottomLimit) {
            // 1. Fully contained
            // topOffset + topLimit <= bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = topSortLimit.fetch;
        } else if (topOffset < bottomLimit) {
            // 2. Partially contained
            // topOffset + topLimit > bottomLimit && topOffset < bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomLimit - topOffset));
        } else {
            // 3. Outside
            // we need to create a new limit 0
            newOffset = null;
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0));
        }
    } else {
        // Bottom operator does not contain offset/fetch
        newOffset = topSortLimit.offset;
        newLimit = topSortLimit.fetch;
    }
    final HiveSortLimit newSort = bottomSortLimit.copy(bottomSortLimit.getTraitSet(), bottomSortLimit.getInput(), bottomSortLimit.collation, newOffset, newLimit);
    call.transformTo(newSort);
}
Also used : RexBuilder(org.apache.calcite.rex.RexBuilder) HiveSortLimit(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit) RexNode(org.apache.calcite.rex.RexNode)

Example 37 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveSortUnionReduceRule method onMatch.

public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit sort = call.rel(0);
    final HiveUnion union = call.rel(1);
    List<RelNode> inputs = new ArrayList<>();
    // Thus we use 'finishPushSortPastUnion' as a flag to identify if we have finished pushing the
    // sort past a union.
    boolean finishPushSortPastUnion = true;
    final int offset = sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
    for (RelNode input : union.getInputs()) {
        // If we do not reduce the input size, we bail out
        if (RexLiteral.intValue(sort.fetch) + offset < RelMetadataQuery.instance().getRowCount(input)) {
            finishPushSortPastUnion = false;
            // Here we do some query rewrite. We first get the new fetchRN, which is
            // a sum of offset and fetch.
            // We then push it through by creating a new branchSort with the new
            // fetchRN but no offset.
            RexNode fetchRN = sort.getCluster().getRexBuilder().makeExactLiteral(BigDecimal.valueOf(RexLiteral.intValue(sort.fetch) + offset));
            HiveSortLimit branchSort = sort.copy(sort.getTraitSet(), input, sort.getCollation(), null, fetchRN);
            branchSort.setRuleCreated(true);
            inputs.add(branchSort);
        } else {
            inputs.add(input);
        }
    }
    // there is nothing to change
    if (finishPushSortPastUnion) {
        return;
    }
    // create new union and sort
    HiveUnion unionCopy = (HiveUnion) union.copy(union.getTraitSet(), inputs, union.all);
    HiveSortLimit result = sort.copy(sort.getTraitSet(), unionCopy, sort.getCollation(), sort.offset, sort.fetch);
    call.transformTo(result);
}
Also used : RelNode(org.apache.calcite.rel.RelNode) ArrayList(java.util.ArrayList) HiveSortLimit(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit) HiveUnion(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion) RexNode(org.apache.calcite.rex.RexNode)

Example 38 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveSubQueryRemoveRule method apply.

protected RexNode apply(RexSubQuery e, Set<CorrelationId> variablesSet, RelOptUtil.Logic logic, HiveSubQRemoveRelBuilder builder, int inputCount, int offset, boolean isCorrScalarAgg) {
    switch(e.getKind()) {
        case SCALAR_QUERY:
            if (isCorrScalarAgg) {
                // Transformation :
                // Outer Query Left Join (inner query) on correlated predicate and preserve rows only from left side.
                builder.push(e.rel);
                final List<RexNode> parentQueryFields = new ArrayList<>();
                parentQueryFields.addAll(builder.fields());
                // id is appended since there could be multiple scalar subqueries and FILTER
                // is created using field name
                String indicator = "alwaysTrue" + e.rel.getId();
                parentQueryFields.add(builder.alias(builder.literal(true), indicator));
                builder.project(parentQueryFields);
                builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
                final ImmutableList.Builder<RexNode> operands = ImmutableList.builder();
                RexNode literal;
                if (isAggZeroOnEmpty(e)) {
                    literal = builder.literal(0);
                } else {
                    literal = e.rel.getCluster().getRexBuilder().makeNullLiteral(getAggTypeForScalarSub(e));
                }
                operands.add((builder.isNull(builder.field(indicator))), literal);
                operands.add(field(builder, 1, builder.fields().size() - 2));
                return builder.call(SqlStdOperatorTable.CASE, operands.build());
            }
            //Transformation is to left join for correlated predicates and inner join otherwise,
            // but do a count on inner side before that to make sure it generates atmost 1 row.
            builder.push(e.rel);
            // returns single row/column
            builder.aggregate(builder.groupKey(), builder.count(false, "cnt"));
            SqlFunction countCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT, InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
            // we create FILTER (sq_count_check(count()) <= 1) instead of PROJECT because RelFieldTrimmer
            //  ends up getting rid of Project since it is not used further up the tree
            builder.filter(builder.call(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, builder.call(countCheck, builder.field("cnt")), builder.literal(1)));
            if (!variablesSet.isEmpty()) {
                builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
            } else
                builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
            builder.push(e.rel);
            builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
            offset++;
            return field(builder, inputCount, offset);
        case IN:
        case EXISTS:
            // Most general case, where the left and right keys might have nulls, and
            // caller requires 3-valued logic return.
            //
            // select e.deptno, e.deptno in (select deptno from emp)
            //
            // becomes
            //
            // select e.deptno,
            //   case
            //   when ct.c = 0 then false
            //   when dt.i is not null then true
            //   when e.deptno is null then null
            //   when ct.ck < ct.c then null
            //   else false
            //   end
            // from e
            // left join (
            //   (select count(*) as c, count(deptno) as ck from emp) as ct
            //   cross join (select distinct deptno, true as i from emp)) as dt
            //   on e.deptno = dt.deptno
            //
            // If keys are not null we can remove "ct" and simplify to
            //
            // select e.deptno,
            //   case
            //   when dt.i is not null then true
            //   else false
            //   end
            // from e
            // left join (select distinct deptno, true as i from emp) as dt
            //   on e.deptno = dt.deptno
            //
            // We could further simplify to
            //
            // select e.deptno,
            //   dt.i is not null
            // from e
            // left join (select distinct deptno, true as i from emp) as dt
            //   on e.deptno = dt.deptno
            //
            // but have not yet.
            //
            // If the logic is TRUE we can just kill the record if the condition
            // evaluates to FALSE or UNKNOWN. Thus the query simplifies to an inner
            // join:
            //
            // select e.deptno,
            //   true
            // from e
            // inner join (select distinct deptno from emp) as dt
            //   on e.deptno = dt.deptno
            //
            builder.push(e.rel);
            final List<RexNode> fields = new ArrayList<>();
            switch(e.getKind()) {
                case IN:
                    fields.addAll(builder.fields());
                    //  will produce wrong results (because we further rewrite such queries into JOIN)
                    if (isCorrScalarAgg) {
                        // returns single row/column
                        builder.aggregate(builder.groupKey(), builder.count(false, "cnt_in"));
                        if (!variablesSet.isEmpty()) {
                            builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
                        } else {
                            builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
                        }
                        SqlFunction inCountCheck = new SqlFunction("sq_count_check", SqlKind.OTHER_FUNCTION, ReturnTypes.BIGINT, InferTypes.RETURN_TYPE, OperandTypes.NUMERIC, SqlFunctionCategory.USER_DEFINED_FUNCTION);
                        // we create FILTER (sq_count_check(count()) > 0) instead of PROJECT because RelFieldTrimmer
                        //  ends up getting rid of Project since it is not used further up the tree
                        builder.filter(builder.call(SqlStdOperatorTable.GREATER_THAN, //true here indicates that sq_count_check is for IN/NOT IN subqueries
                        builder.call(inCountCheck, builder.field("cnt_in"), builder.literal(true)), builder.literal(0)));
                        offset = offset + 1;
                        builder.push(e.rel);
                    }
            }
            // First, the cross join
            switch(logic) {
                case TRUE_FALSE_UNKNOWN:
                case UNKNOWN_AS_TRUE:
                    // null keys we do not need to generate count(*), count(c)
                    if (e.getKind() == SqlKind.EXISTS) {
                        logic = RelOptUtil.Logic.TRUE_FALSE;
                        break;
                    }
                    builder.aggregate(builder.groupKey(), builder.count(false, "c"), builder.aggregateCall(SqlStdOperatorTable.COUNT, false, null, "ck", builder.fields()));
                    builder.as("ct");
                    if (!variablesSet.isEmpty()) {
                        //builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
                        builder.join(JoinRelType.LEFT, builder.literal(true), variablesSet);
                    } else
                        builder.join(JoinRelType.INNER, builder.literal(true), variablesSet);
                    offset += 2;
                    builder.push(e.rel);
                    break;
            }
            // Now the left join
            switch(logic) {
                case TRUE:
                    if (fields.isEmpty()) {
                        builder.project(builder.alias(builder.literal(true), "i" + e.rel.getId()));
                        builder.aggregate(builder.groupKey(0));
                    } else {
                        builder.aggregate(builder.groupKey(fields));
                    }
                    break;
                default:
                    fields.add(builder.alias(builder.literal(true), "i" + e.rel.getId()));
                    builder.project(fields);
                    builder.distinct();
            }
            builder.as("dt");
            final List<RexNode> conditions = new ArrayList<>();
            for (Pair<RexNode, RexNode> pair : Pair.zip(e.getOperands(), builder.fields())) {
                conditions.add(builder.equals(pair.left, RexUtil.shift(pair.right, offset)));
            }
            switch(logic) {
                case TRUE:
                    builder.join(JoinRelType.INNER, builder.and(conditions), variablesSet);
                    return builder.literal(true);
            }
            builder.join(JoinRelType.LEFT, builder.and(conditions), variablesSet);
            final List<RexNode> keyIsNulls = new ArrayList<>();
            for (RexNode operand : e.getOperands()) {
                if (operand.getType().isNullable()) {
                    keyIsNulls.add(builder.isNull(operand));
                }
            }
            final ImmutableList.Builder<RexNode> operands = ImmutableList.builder();
            switch(logic) {
                case TRUE_FALSE_UNKNOWN:
                case UNKNOWN_AS_TRUE:
                    operands.add(builder.equals(builder.field("ct", "c"), builder.literal(0)), builder.literal(false));
                    //now that we are using LEFT OUTER JOIN to join inner count, count(*)
                    // with outer table, we wouldn't be able to tell if count is zero
                    // for inner table since inner join with correlated values will get rid
                    // of all values where join cond is not true (i.e where actual inner table
                    // will produce zero result). To  handle this case we need to check both
                    // count is zero or count is null
                    operands.add((builder.isNull(builder.field("ct", "c"))), builder.literal(false));
                    break;
            }
            operands.add(builder.isNotNull(builder.field("dt", "i" + e.rel.getId())), builder.literal(true));
            if (!keyIsNulls.isEmpty()) {
                //Calcite creates null literal with Null type here but because HIVE doesn't support null type
                // it is appropriately typed boolean
                operands.add(builder.or(keyIsNulls), e.rel.getCluster().getRexBuilder().makeNullLiteral(SqlTypeName.BOOLEAN));
            // we are creating filter here so should not be returning NULL. Not sure why Calcite return NULL
            //operands.add(builder.or(keyIsNulls), builder.literal(false));
            }
            RexNode b = builder.literal(true);
            switch(logic) {
                case TRUE_FALSE_UNKNOWN:
                    b = e.rel.getCluster().getRexBuilder().makeNullLiteral(SqlTypeName.BOOLEAN);
                // fall through
                case UNKNOWN_AS_TRUE:
                    operands.add(builder.call(SqlStdOperatorTable.LESS_THAN, builder.field("ct", "ck"), builder.field("ct", "c")), b);
                    break;
            }
            operands.add(builder.literal(false));
            return builder.call(SqlStdOperatorTable.CASE, operands.build());
        default:
            throw new AssertionError(e.getKind());
    }
}
Also used : ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) RexNode(org.apache.calcite.rex.RexNode) SqlFunction(org.apache.calcite.sql.SqlFunction)

Example 39 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveUnionPullUpConstantsRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Union union = call.rel(0);
    final int count = union.getRowType().getFieldCount();
    if (count == 1) {
        // Project operator.
        return;
    }
    final RexBuilder rexBuilder = union.getCluster().getRexBuilder();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(union);
    if (predicates == null) {
        return;
    }
    Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
    Map<RexNode, RexNode> constants = new HashMap<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        if (conditionsExtracted.containsKey(expr)) {
            constants.put(expr, conditionsExtracted.get(expr));
        }
    }
    // None of the expressions are constant. Nothing to do.
    if (constants.isEmpty()) {
        return;
    }
    // Create expressions for Project operators before and after the Union
    List<RelDataTypeField> fields = union.getRowType().getFieldList();
    List<RexNode> topChildExprs = new ArrayList<>();
    List<String> topChildExprsFields = new ArrayList<>();
    List<RexNode> refs = new ArrayList<>();
    ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        RelDataTypeField field = fields.get(i);
        if (constants.containsKey(expr)) {
            topChildExprs.add(constants.get(expr));
            topChildExprsFields.add(field.getName());
        } else {
            topChildExprs.add(expr);
            topChildExprsFields.add(field.getName());
            refs.add(expr);
            refsIndexBuilder.set(i);
        }
    }
    ImmutableBitSet refsIndex = refsIndexBuilder.build();
    // Update top Project positions
    final Mappings.TargetMapping mapping = RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse();
    topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
    // Create new Project-Union-Project sequences
    final RelBuilder relBuilder = call.builder();
    for (int i = 0; i < union.getInputs().size(); i++) {
        RelNode input = union.getInput(i);
        List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
        for (int j = 0; j < refsIndex.cardinality(); j++) {
            int pos = refsIndex.nth(j);
            newChildExprs.add(Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos), input.getRowType().getFieldList().get(pos).getName()));
        }
        if (newChildExprs.isEmpty()) {
            // At least a single item in project is required.
            newChildExprs.add(Pair.<RexNode, String>of(topChildExprs.get(0), topChildExprsFields.get(0)));
        }
        // Add the input with project on top
        relBuilder.push(input);
        relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
    }
    relBuilder.union(union.all, union.getInputs().size());
    // Create top Project fixing nullability of fields
    relBuilder.project(topChildExprs, topChildExprsFields);
    relBuilder.convert(union.getRowType(), false);
    call.transformTo(relBuilder.build());
}
Also used : RelMetadataQuery(org.apache.calcite.rel.metadata.RelMetadataQuery) RelBuilder(org.apache.calcite.tools.RelBuilder) ImmutableBitSet(org.apache.calcite.util.ImmutableBitSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HiveUnion(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion) Union(org.apache.calcite.rel.core.Union) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) Mappings(org.apache.calcite.util.mapping.Mappings) RelNode(org.apache.calcite.rel.RelNode) RelOptPredicateList(org.apache.calcite.plan.RelOptPredicateList) RexBuilder(org.apache.calcite.rex.RexBuilder) RexNode(org.apache.calcite.rex.RexNode) Pair(org.apache.calcite.util.Pair)

Example 40 with RexNode

use of org.apache.calcite.rex.RexNode in project hive by apache.

the class HiveAggregateJoinTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final Join join = call.rel(1);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    // If any aggregate call has a filter, bail out
    for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
        if (aggregateCall.getAggregation().unwrap(SqlSplittableAggFunction.class) == null) {
            return;
        }
        if (aggregateCall.filterArg >= 0) {
            return;
        }
    }
    // aggregate operator
    if (join.getJoinType() != JoinRelType.INNER) {
        return;
    }
    if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) {
        return;
    }
    // Do the columns used by the join appear in the output of the aggregate?
    RelMetadataQuery mq = RelMetadataQuery.instance();
    final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
    final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates);
    final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition());
    final boolean allColumnsInAggregate = keyColumns.contains(joinColumns);
    final ImmutableBitSet belowAggregateColumns = aggregateColumns.union(joinColumns);
    // Split join condition
    final List<Integer> leftKeys = Lists.newArrayList();
    final List<Integer> rightKeys = Lists.newArrayList();
    final List<Boolean> filterNulls = Lists.newArrayList();
    RexNode nonEquiConj = RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(), join.getCondition(), leftKeys, rightKeys, filterNulls);
    // If it contains non-equi join conditions, we bail out
    if (!nonEquiConj.isAlwaysTrue()) {
        return;
    }
    // Push each aggregate function down to each side that contains all of its
    // arguments. Note that COUNT(*), because it has no arguments, can go to
    // both sides.
    final Map<Integer, Integer> map = new HashMap<>();
    final List<Side> sides = new ArrayList<>();
    int uniqueCount = 0;
    int offset = 0;
    int belowOffset = 0;
    for (int s = 0; s < 2; s++) {
        final Side side = new Side();
        final RelNode joinInput = join.getInput(s);
        int fieldCount = joinInput.getRowType().getFieldCount();
        final ImmutableBitSet fieldSet = ImmutableBitSet.range(offset, offset + fieldCount);
        final ImmutableBitSet belowAggregateKeyNotShifted = belowAggregateColumns.intersect(fieldSet);
        for (Ord<Integer> c : Ord.zip(belowAggregateKeyNotShifted)) {
            map.put(c.e, belowOffset + c.i);
        }
        final ImmutableBitSet belowAggregateKey = belowAggregateKeyNotShifted.shift(-offset);
        final boolean unique;
        if (!allowFunctions) {
            assert aggregate.getAggCallList().isEmpty();
            // If there are no functions, it doesn't matter as much whether we
            // aggregate the inputs before the join, because there will not be
            // any functions experiencing a cartesian product effect.
            //
            // But finding out whether the input is already unique requires a call
            // to areColumnsUnique that currently (until [CALCITE-794] "Detect
            // cycles when computing statistics" is fixed) places a heavy load on
            // the metadata system.
            //
            // So we choose to imagine the the input is already unique, which is
            // untrue but harmless.
            //
            unique = true;
        } else {
            final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey);
            unique = unique0 != null && unique0;
        }
        if (unique) {
            ++uniqueCount;
            side.newInput = joinInput;
        } else {
            List<AggregateCall> belowAggCalls = new ArrayList<>();
            final SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry = registry(belowAggCalls);
            final Mappings.TargetMapping mapping = s == 0 ? Mappings.createIdentity(fieldCount) : Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
            for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
                final SqlAggFunction aggregation = aggCall.e.getAggregation();
                final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
                final AggregateCall call1;
                if (fieldSet.contains(ImmutableBitSet.of(aggCall.e.getArgList()))) {
                    call1 = splitter.split(aggCall.e, mapping);
                } else {
                    call1 = splitter.other(rexBuilder.getTypeFactory(), aggCall.e);
                }
                if (call1 != null) {
                    side.split.put(aggCall.i, belowAggregateKey.cardinality() + belowAggCallRegistry.register(call1));
                }
            }
            side.newInput = aggregateFactory.createAggregate(joinInput, false, belowAggregateKey, null, belowAggCalls);
        }
        offset += fieldCount;
        belowOffset += side.newInput.getRowType().getFieldCount();
        sides.add(side);
    }
    if (uniqueCount == 2) {
        // invocation of this rule; if we continue we might loop forever.
        return;
    }
    // Update condition
    final Mapping mapping = (Mapping) Mappings.target(new Function<Integer, Integer>() {

        @Override
        public Integer apply(Integer a0) {
            return map.get(a0);
        }
    }, join.getRowType().getFieldCount(), belowOffset);
    final RexNode newCondition = RexUtil.apply(mapping, join.getCondition());
    // Create new join
    RelNode newJoin = joinFactory.createJoin(sides.get(0).newInput, sides.get(1).newInput, newCondition, join.getJoinType(), join.getVariablesStopped(), join.isSemiJoinDone());
    // Aggregate above to sum up the sub-totals
    final List<AggregateCall> newAggCalls = new ArrayList<>();
    final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
    final List<RexNode> projects = new ArrayList<>(rexBuilder.identityProjects(newJoin.getRowType()));
    for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
        final SqlAggFunction aggregation = aggCall.e.getAggregation();
        final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
        final Integer leftSubTotal = sides.get(0).split.get(aggCall.i);
        final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
        newAggCalls.add(splitter.topSplit(rexBuilder, registry(projects), groupIndicatorCount, newJoin.getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
    }
    RelNode r = newJoin;
    b: if (allColumnsInAggregate && newAggCalls.isEmpty() && RelOptUtil.areRowTypesEqual(r.getRowType(), aggregate.getRowType(), false)) {
    // no need to aggregate
    } else {
        r = RelOptUtil.createProject(r, projects, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
        if (allColumnsInAggregate) {
            // let's see if we can convert
            List<RexNode> projects2 = new ArrayList<>();
            for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
                projects2.add(rexBuilder.makeInputRef(r, key));
            }
            for (AggregateCall newAggCall : newAggCalls) {
                final SqlSplittableAggFunction splitter = newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
                if (splitter != null) {
                    projects2.add(splitter.singleton(rexBuilder, r.getRowType(), newAggCall));
                }
            }
            if (projects2.size() == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
                // We successfully converted agg calls into projects.
                r = RelOptUtil.createProject(r, projects2, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
                break b;
            }
        }
        r = aggregateFactory.createAggregate(r, aggregate.indicator, Mappings.apply(mapping, aggregate.getGroupSet()), Mappings.apply2(mapping, aggregate.getGroupSets()), newAggCalls);
    }
    // Make a cost based decision to pick cheaper plan
    RelOptCost afterCost = mq.getCumulativeCost(r);
    RelOptCost beforeCost = mq.getCumulativeCost(aggregate);
    if (afterCost.isLt(beforeCost)) {
        call.transformTo(r);
    }
}
Also used : RelMetadataQuery(org.apache.calcite.rel.metadata.RelMetadataQuery) ImmutableBitSet(org.apache.calcite.util.ImmutableBitSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Mapping(org.apache.calcite.util.mapping.Mapping) Function(com.google.common.base.Function) SqlSplittableAggFunction(org.apache.calcite.sql.SqlSplittableAggFunction) SqlAggFunction(org.apache.calcite.sql.SqlAggFunction) RelOptCost(org.apache.calcite.plan.RelOptCost) RexBuilder(org.apache.calcite.rex.RexBuilder) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) SqlSplittableAggFunction(org.apache.calcite.sql.SqlSplittableAggFunction) Join(org.apache.calcite.rel.core.Join) HiveJoin(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin) SqlAggFunction(org.apache.calcite.sql.SqlAggFunction) AggregateCall(org.apache.calcite.rel.core.AggregateCall) RelNode(org.apache.calcite.rel.RelNode) Mappings(org.apache.calcite.util.mapping.Mappings) HiveAggregate(org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate) Aggregate(org.apache.calcite.rel.core.Aggregate) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

RexNode (org.apache.calcite.rex.RexNode)204 RelNode (org.apache.calcite.rel.RelNode)75 ArrayList (java.util.ArrayList)68 RexBuilder (org.apache.calcite.rex.RexBuilder)52 RelDataType (org.apache.calcite.rel.type.RelDataType)48 RexInputRef (org.apache.calcite.rex.RexInputRef)43 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)41 RexCall (org.apache.calcite.rex.RexCall)32 ImmutableBitSet (org.apache.calcite.util.ImmutableBitSet)28 Pair (org.apache.calcite.util.Pair)22 HashMap (java.util.HashMap)21 AggregateCall (org.apache.calcite.rel.core.AggregateCall)21 RexLiteral (org.apache.calcite.rex.RexLiteral)19 ImmutableList (com.google.common.collect.ImmutableList)18 Project (org.apache.calcite.rel.core.Project)17 RelDataTypeFactory (org.apache.calcite.rel.type.RelDataTypeFactory)14 Filter (org.apache.calcite.rel.core.Filter)12 HashSet (java.util.HashSet)11 CalciteSemanticException (org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException)11 BigDecimal (java.math.BigDecimal)10