Examples with RexBuilder org.apache.calcite.rex.RexBuilder used on opensource projects

Example 11 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveSortLimitPullUpConstantsRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final RelNode parent = call.rel(0);
    final Sort sort = call.rel(1);
    final int count = sort.getInput().getRowType().getFieldCount();
    if (count == 1) {
        // Project operator.
        return;
    }
    final RexBuilder rexBuilder = sort.getCluster().getRexBuilder();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(sort.getInput());
    if (predicates == null) {
        return;
    }
    Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
    Map<RexNode, RexNode> constants = new HashMap<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
        if (conditionsExtracted.containsKey(expr)) {
            constants.put(expr, conditionsExtracted.get(expr));
        }
    }
    // None of the expressions are constant. Nothing to do.
    if (constants.isEmpty()) {
        return;
    }
    if (count == constants.size()) {
        // At least a single item in project is required.
        constants.remove(constants.keySet().iterator().next());
    }
    // Create expressions for Project operators before and after the Sort
    List<RelDataTypeField> fields = sort.getInput().getRowType().getFieldList();
    List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
    List<RexNode> topChildExprs = new ArrayList<>();
    List<String> topChildExprsFields = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(sort.getInput(), i);
        RelDataTypeField field = fields.get(i);
        if (constants.containsKey(expr)) {
            topChildExprs.add(constants.get(expr));
            topChildExprsFields.add(field.getName());
        } else {
            newChildExprs.add(Pair.<RexNode, String>of(expr, field.getName()));
            topChildExprs.add(expr);
            topChildExprsFields.add(field.getName());
        }
    }
    // Update field collations
    final Mappings.TargetMapping mapping = RelOptUtil.permutation(Pair.left(newChildExprs), sort.getInput().getRowType()).inverse();
    List<RelFieldCollation> fieldCollations = new ArrayList<>();
    for (RelFieldCollation fc : sort.getCollation().getFieldCollations()) {
        final int target = mapping.getTargetOpt(fc.getFieldIndex());
        if (target < 0) {
            // It is a constant, we can ignore it
            continue;
        }
        fieldCollations.add(fc.copy(target));
    }
    // Update top Project positions
    topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
    // Create new Project-Sort-Project sequence
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(sort.getInput());
    relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
    final ImmutableList<RexNode> sortFields = relBuilder.fields(RelCollations.of(fieldCollations));
    relBuilder.sortLimit(sort.offset == null ? -1 : RexLiteral.intValue(sort.offset), sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch), sortFields);
    // Create top Project fixing nullability of fields
    relBuilder.project(topChildExprs, topChildExprsFields);
    relBuilder.convert(sort.getRowType(), false);
    List<RelNode> inputs = new ArrayList<>();
    for (RelNode child : parent.getInputs()) {
        if (!((HepRelVertex) child).getCurrentRel().equals(sort)) {
            inputs.add(child);
        } else {
            inputs.add(relBuilder.build());
        }
    }
    call.transformTo(parent.copy(parent.getTraitSet(), inputs));
}

Example 12 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveSortMergeRule method onMatch.

// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final HiveSortLimit topSortLimit = call.rel(0);
    final HiveSortLimit bottomSortLimit = call.rel(1);
    final RexNode newOffset;
    final RexNode newLimit;
    if (HiveCalciteUtil.limitRelNode(bottomSortLimit)) {
        final RexBuilder rexBuilder = topSortLimit.getCluster().getRexBuilder();
        int topOffset = topSortLimit.offset == null ? 0 : RexLiteral.intValue(topSortLimit.offset);
        int topLimit = RexLiteral.intValue(topSortLimit.fetch);
        int bottomOffset = bottomSortLimit.offset == null ? 0 : RexLiteral.intValue(bottomSortLimit.offset);
        int bottomLimit = RexLiteral.intValue(bottomSortLimit.fetch);
        // Three different cases
        if (topOffset + topLimit <= bottomLimit) {
            // 1. Fully contained
            // topOffset + topLimit <= bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = topSortLimit.fetch;
        } else if (topOffset < bottomLimit) {
            // 2. Partially contained
            // topOffset + topLimit > bottomLimit && topOffset < bottomLimit
            newOffset = bottomOffset + topOffset == 0 ? null : rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomOffset + topOffset));
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(bottomLimit - topOffset));
        } else {
            // 3. Outside
            // we need to create a new limit 0
            newOffset = null;
            newLimit = rexBuilder.makeExactLiteral(BigDecimal.valueOf(0));
        }
    } else {
        // Bottom operator does not contain offset/fetch
        newOffset = topSortLimit.offset;
        newLimit = topSortLimit.fetch;
    }
    final HiveSortLimit newSort = bottomSortLimit.copy(bottomSortLimit.getTraitSet(), bottomSortLimit.getInput(), bottomSortLimit.collation, newOffset, newLimit);
    call.transformTo(newSort);
}

Example 13 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveUnionPullUpConstantsRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Union union = call.rel(0);
    final int count = union.getRowType().getFieldCount();
    if (count == 1) {
        // Project operator.
        return;
    }
    final RexBuilder rexBuilder = union.getCluster().getRexBuilder();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(union);
    if (predicates == null) {
        return;
    }
    Map<RexNode, RexNode> conditionsExtracted = HiveReduceExpressionsRule.predicateConstants(RexNode.class, rexBuilder, predicates);
    Map<RexNode, RexNode> constants = new HashMap<>();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        if (conditionsExtracted.containsKey(expr)) {
            constants.put(expr, conditionsExtracted.get(expr));
        }
    }
    // None of the expressions are constant. Nothing to do.
    if (constants.isEmpty()) {
        return;
    }
    // Create expressions for Project operators before and after the Union
    List<RelDataTypeField> fields = union.getRowType().getFieldList();
    List<RexNode> topChildExprs = new ArrayList<>();
    List<String> topChildExprsFields = new ArrayList<>();
    List<RexNode> refs = new ArrayList<>();
    ImmutableBitSet.Builder refsIndexBuilder = ImmutableBitSet.builder();
    for (int i = 0; i < count; i++) {
        RexNode expr = rexBuilder.makeInputRef(union, i);
        RelDataTypeField field = fields.get(i);
        if (constants.containsKey(expr)) {
            topChildExprs.add(constants.get(expr));
            topChildExprsFields.add(field.getName());
        } else {
            topChildExprs.add(expr);
            topChildExprsFields.add(field.getName());
            refs.add(expr);
            refsIndexBuilder.set(i);
        }
    }
    ImmutableBitSet refsIndex = refsIndexBuilder.build();
    // Update top Project positions
    final Mappings.TargetMapping mapping = RelOptUtil.permutation(refs, union.getInput(0).getRowType()).inverse();
    topChildExprs = ImmutableList.copyOf(RexUtil.apply(mapping, topChildExprs));
    // Create new Project-Union-Project sequences
    final RelBuilder relBuilder = call.builder();
    for (int i = 0; i < union.getInputs().size(); i++) {
        RelNode input = union.getInput(i);
        List<Pair<RexNode, String>> newChildExprs = new ArrayList<>();
        for (int j = 0; j < refsIndex.cardinality(); j++) {
            int pos = refsIndex.nth(j);
            newChildExprs.add(Pair.<RexNode, String>of(rexBuilder.makeInputRef(input, pos), input.getRowType().getFieldList().get(pos).getName()));
        }
        if (newChildExprs.isEmpty()) {
            // At least a single item in project is required.
            newChildExprs.add(Pair.<RexNode, String>of(topChildExprs.get(0), topChildExprsFields.get(0)));
        }
        // Add the input with project on top
        relBuilder.push(input);
        relBuilder.project(Pair.left(newChildExprs), Pair.right(newChildExprs));
    }
    relBuilder.union(union.all, union.getInputs().size());
    // Create top Project fixing nullability of fields
    relBuilder.project(topChildExprs, topChildExprsFields);
    relBuilder.convert(union.getRowType(), false);
    call.transformTo(relBuilder.build());
}

Example 14 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveAggregateJoinTransposeRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final Join join = call.rel(1);
    final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    // If any aggregate call has a filter, bail out
    for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
        if (aggregateCall.getAggregation().unwrap(SqlSplittableAggFunction.class) == null) {
            return;
        }
        if (aggregateCall.filterArg >= 0) {
            return;
        }
    }
    // aggregate operator
    if (join.getJoinType() != JoinRelType.INNER) {
        return;
    }
    if (!allowFunctions && !aggregate.getAggCallList().isEmpty()) {
        return;
    }
    // Do the columns used by the join appear in the output of the aggregate?
    RelMetadataQuery mq = RelMetadataQuery.instance();
    final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
    final ImmutableBitSet keyColumns = keyColumns(aggregateColumns, mq.getPulledUpPredicates(join).pulledUpPredicates);
    final ImmutableBitSet joinColumns = RelOptUtil.InputFinder.bits(join.getCondition());
    final boolean allColumnsInAggregate = keyColumns.contains(joinColumns);
    final ImmutableBitSet belowAggregateColumns = aggregateColumns.union(joinColumns);
    // Split join condition
    final List<Integer> leftKeys = Lists.newArrayList();
    final List<Integer> rightKeys = Lists.newArrayList();
    final List<Boolean> filterNulls = Lists.newArrayList();
    RexNode nonEquiConj = RelOptUtil.splitJoinCondition(join.getLeft(), join.getRight(), join.getCondition(), leftKeys, rightKeys, filterNulls);
    // If it contains non-equi join conditions, we bail out
    if (!nonEquiConj.isAlwaysTrue()) {
        return;
    }
    // Push each aggregate function down to each side that contains all of its
    // arguments. Note that COUNT(*), because it has no arguments, can go to
    // both sides.
    final Map<Integer, Integer> map = new HashMap<>();
    final List<Side> sides = new ArrayList<>();
    int uniqueCount = 0;
    int offset = 0;
    int belowOffset = 0;
    for (int s = 0; s < 2; s++) {
        final Side side = new Side();
        final RelNode joinInput = join.getInput(s);
        int fieldCount = joinInput.getRowType().getFieldCount();
        final ImmutableBitSet fieldSet = ImmutableBitSet.range(offset, offset + fieldCount);
        final ImmutableBitSet belowAggregateKeyNotShifted = belowAggregateColumns.intersect(fieldSet);
        for (Ord<Integer> c : Ord.zip(belowAggregateKeyNotShifted)) {
            map.put(c.e, belowOffset + c.i);
        }
        final ImmutableBitSet belowAggregateKey = belowAggregateKeyNotShifted.shift(-offset);
        final boolean unique;
        if (!allowFunctions) {
            assert aggregate.getAggCallList().isEmpty();
            // If there are no functions, it doesn't matter as much whether we
            // aggregate the inputs before the join, because there will not be
            // any functions experiencing a cartesian product effect.
            //
            // But finding out whether the input is already unique requires a call
            // to areColumnsUnique that currently (until [CALCITE-794] "Detect
            // cycles when computing statistics" is fixed) places a heavy load on
            // the metadata system.
            //
            // So we choose to imagine the the input is already unique, which is
            // untrue but harmless.
            //
            unique = true;
        } else {
            final Boolean unique0 = mq.areColumnsUnique(joinInput, belowAggregateKey);
            unique = unique0 != null && unique0;
        }
        if (unique) {
            ++uniqueCount;
            side.newInput = joinInput;
        } else {
            List<AggregateCall> belowAggCalls = new ArrayList<>();
            final SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry = registry(belowAggCalls);
            final Mappings.TargetMapping mapping = s == 0 ? Mappings.createIdentity(fieldCount) : Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
            for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
                final SqlAggFunction aggregation = aggCall.e.getAggregation();
                final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
                final AggregateCall call1;
                if (fieldSet.contains(ImmutableBitSet.of(aggCall.e.getArgList()))) {
                    call1 = splitter.split(aggCall.e, mapping);
                } else {
                    call1 = splitter.other(rexBuilder.getTypeFactory(), aggCall.e);
                }
                if (call1 != null) {
                    side.split.put(aggCall.i, belowAggregateKey.cardinality() + belowAggCallRegistry.register(call1));
                }
            }
            side.newInput = aggregateFactory.createAggregate(joinInput, false, belowAggregateKey, null, belowAggCalls);
        }
        offset += fieldCount;
        belowOffset += side.newInput.getRowType().getFieldCount();
        sides.add(side);
    }
    if (uniqueCount == 2) {
        // invocation of this rule; if we continue we might loop forever.
        return;
    }
    // Update condition
    final Mapping mapping = (Mapping) Mappings.target(new Function<Integer, Integer>() {

        @Override
        public Integer apply(Integer a0) {
            return map.get(a0);
        }
    }, join.getRowType().getFieldCount(), belowOffset);
    final RexNode newCondition = RexUtil.apply(mapping, join.getCondition());
    // Create new join
    RelNode newJoin = joinFactory.createJoin(sides.get(0).newInput, sides.get(1).newInput, newCondition, join.getJoinType(), join.getVariablesStopped(), join.isSemiJoinDone());
    // Aggregate above to sum up the sub-totals
    final List<AggregateCall> newAggCalls = new ArrayList<>();
    final int groupIndicatorCount = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
    final List<RexNode> projects = new ArrayList<>(rexBuilder.identityProjects(newJoin.getRowType()));
    for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
        final SqlAggFunction aggregation = aggCall.e.getAggregation();
        final SqlSplittableAggFunction splitter = Preconditions.checkNotNull(aggregation.unwrap(SqlSplittableAggFunction.class));
        final Integer leftSubTotal = sides.get(0).split.get(aggCall.i);
        final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
        newAggCalls.add(splitter.topSplit(rexBuilder, registry(projects), groupIndicatorCount, newJoin.getRowType(), aggCall.e, leftSubTotal == null ? -1 : leftSubTotal, rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
    }
    RelNode r = newJoin;
    b: if (allColumnsInAggregate && newAggCalls.isEmpty() && RelOptUtil.areRowTypesEqual(r.getRowType(), aggregate.getRowType(), false)) {
    // no need to aggregate
    } else {
        r = RelOptUtil.createProject(r, projects, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
        if (allColumnsInAggregate) {
            // let's see if we can convert
            List<RexNode> projects2 = new ArrayList<>();
            for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
                projects2.add(rexBuilder.makeInputRef(r, key));
            }
            for (AggregateCall newAggCall : newAggCalls) {
                final SqlSplittableAggFunction splitter = newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
                if (splitter != null) {
                    projects2.add(splitter.singleton(rexBuilder, r.getRowType(), newAggCall));
                }
            }
            if (projects2.size() == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
                // We successfully converted agg calls into projects.
                r = RelOptUtil.createProject(r, projects2, null, true, relBuilderFactory.create(aggregate.getCluster(), null));
                break b;
            }
        }
        r = aggregateFactory.createAggregate(r, aggregate.indicator, Mappings.apply(mapping, aggregate.getGroupSet()), Mappings.apply2(mapping, aggregate.getGroupSets()), newAggCalls);
    }
    // Make a cost based decision to pick cheaper plan
    RelOptCost afterCost = mq.getCumulativeCost(r);
    RelOptCost beforeCost = mq.getCumulativeCost(aggregate);
    if (afterCost.isLt(beforeCost)) {
        call.transformTo(r);
    }
}

Example 15 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveExceptRewriteRule method makeFilterExprForExceptDistinct.

private RexNode makeFilterExprForExceptDistinct(HiveRelNode input, int columnSize, RelOptCluster cluster, RexBuilder rexBuilder) throws CalciteSemanticException {
    List<RexNode> childRexNodeLst = new ArrayList<RexNode>();
    RexInputRef a = rexBuilder.makeInputRef(input, columnSize - 2);
    RexLiteral zero = rexBuilder.makeBigintLiteral(new BigDecimal(0));
    childRexNodeLst.add(a);
    childRexNodeLst.add(zero);
    ImmutableList.Builder<RelDataType> calciteArgTypesBldr = new ImmutableList.Builder<RelDataType>();
    calciteArgTypesBldr.add(TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()));
    calciteArgTypesBldr.add(TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()));
    // a>0
    RexNode aMorethanZero = rexBuilder.makeCall(SqlFunctionConverter.getCalciteFn(">", calciteArgTypesBldr.build(), TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()), false), childRexNodeLst);
    childRexNodeLst = new ArrayList<RexNode>();
    RexLiteral two = rexBuilder.makeBigintLiteral(new BigDecimal(2));
    childRexNodeLst.add(a);
    childRexNodeLst.add(two);
    // 2*a
    RexNode twoa = rexBuilder.makeCall(SqlFunctionConverter.getCalciteFn("*", calciteArgTypesBldr.build(), TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()), false), childRexNodeLst);
    childRexNodeLst = new ArrayList<RexNode>();
    RexInputRef b = rexBuilder.makeInputRef(input, columnSize - 1);
    childRexNodeLst.add(twoa);
    childRexNodeLst.add(b);
    // 2a=b
    RexNode twoaEqualTob = rexBuilder.makeCall(SqlFunctionConverter.getCalciteFn("=", calciteArgTypesBldr.build(), TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()), false), childRexNodeLst);
    childRexNodeLst = new ArrayList<RexNode>();
    childRexNodeLst.add(aMorethanZero);
    childRexNodeLst.add(twoaEqualTob);
    // a>0 && 2a=b
    return rexBuilder.makeCall(SqlFunctionConverter.getCalciteFn("and", calciteArgTypesBldr.build(), TypeConverter.convert(TypeInfoFactory.longTypeInfo, cluster.getTypeFactory()), false), childRexNodeLst);
}