Examples with RexBuilder org.apache.calcite.rex.RexBuilder used on opensource projects

Example 46 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveRelOptUtil method splitJoinCondition.

private static void splitJoinCondition(List<RelDataTypeField> sysFieldList, List<RelNode> inputs, RexNode condition, List<List<RexNode>> joinKeys, List<Integer> filterNulls, List<SqlOperator> rangeOp, List<RexNode> nonEquiList) throws CalciteSemanticException {
    final int sysFieldCount = sysFieldList.size();
    final RelOptCluster cluster = inputs.get(0).getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();
    if (condition instanceof RexCall) {
        RexCall call = (RexCall) condition;
        if (call.getOperator() == SqlStdOperatorTable.AND) {
            for (RexNode operand : call.getOperands()) {
                splitJoinCondition(sysFieldList, inputs, operand, joinKeys, filterNulls, rangeOp, nonEquiList);
            }
            return;
        }
        RexNode leftKey = null;
        RexNode rightKey = null;
        int leftInput = 0;
        int rightInput = 0;
        List<RelDataTypeField> leftFields = null;
        List<RelDataTypeField> rightFields = null;
        boolean reverse = false;
        SqlKind kind = call.getKind();
        // Only consider range operators if we haven't already seen one
        if ((kind == SqlKind.EQUALS) || (filterNulls != null && kind == SqlKind.IS_NOT_DISTINCT_FROM) || (rangeOp != null && rangeOp.isEmpty() && (kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL))) {
            final List<RexNode> operands = call.getOperands();
            RexNode op0 = operands.get(0);
            RexNode op1 = operands.get(1);
            final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
            final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
            final ImmutableBitSet[] inputsRange = new ImmutableBitSet[inputs.size()];
            int totalFieldCount = 0;
            for (int i = 0; i < inputs.size(); i++) {
                final int firstField = totalFieldCount + sysFieldCount;
                totalFieldCount = firstField + inputs.get(i).getRowType().getFieldCount();
                inputsRange[i] = ImmutableBitSet.range(firstField, totalFieldCount);
            }
            boolean foundBothInputs = false;
            for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
                if (projRefs0.intersects(inputsRange[i]) && projRefs0.union(inputsRange[i]).equals(inputsRange[i])) {
                    if (leftKey == null) {
                        leftKey = op0;
                        leftInput = i;
                        leftFields = inputs.get(leftInput).getRowType().getFieldList();
                    } else {
                        rightKey = op0;
                        rightInput = i;
                        rightFields = inputs.get(rightInput).getRowType().getFieldList();
                        reverse = true;
                        foundBothInputs = true;
                    }
                } else if (projRefs1.intersects(inputsRange[i]) && projRefs1.union(inputsRange[i]).equals(inputsRange[i])) {
                    if (leftKey == null) {
                        leftKey = op1;
                        leftInput = i;
                        leftFields = inputs.get(leftInput).getRowType().getFieldList();
                    } else {
                        rightKey = op1;
                        rightInput = i;
                        rightFields = inputs.get(rightInput).getRowType().getFieldList();
                        foundBothInputs = true;
                    }
                }
            }
            if ((leftKey != null) && (rightKey != null)) {
                // adjustment array
                int[] adjustments = new int[totalFieldCount];
                for (int i = 0; i < inputs.size(); i++) {
                    final int adjustment = inputsRange[i].nextSetBit(0);
                    for (int j = adjustment; j < inputsRange[i].length(); j++) {
                        adjustments[j] = -adjustment;
                    }
                }
                // replace right Key input ref
                rightKey = rightKey.accept(new RelOptUtil.RexInputConverter(rexBuilder, rightFields, rightFields, adjustments));
                // left key only needs to be adjusted if there are system
                // fields, but do it for uniformity
                leftKey = leftKey.accept(new RelOptUtil.RexInputConverter(rexBuilder, leftFields, leftFields, adjustments));
                RelDataType leftKeyType = leftKey.getType();
                RelDataType rightKeyType = rightKey.getType();
                if (leftKeyType != rightKeyType) {
                    // perform casting using Hive rules
                    TypeInfo rType = TypeConverter.convert(rightKeyType);
                    TypeInfo lType = TypeConverter.convert(leftKeyType);
                    TypeInfo tgtType = FunctionRegistry.getCommonClassForComparison(lType, rType);
                    if (tgtType == null) {
                        throw new CalciteSemanticException("Cannot find common type for join keys " + leftKey + " (type " + leftKeyType + ") and " + rightKey + " (type " + rightKeyType + ")");
                    }
                    RelDataType targetKeyType = TypeConverter.convert(tgtType, rexBuilder.getTypeFactory());
                    if (leftKeyType != targetKeyType && TypeInfoUtils.isConversionRequiredForComparison(tgtType, lType)) {
                        leftKey = rexBuilder.makeCast(targetKeyType, leftKey);
                    }
                    if (rightKeyType != targetKeyType && TypeInfoUtils.isConversionRequiredForComparison(tgtType, rType)) {
                        rightKey = rexBuilder.makeCast(targetKeyType, rightKey);
                    }
                }
            }
        }
        if ((leftKey != null) && (rightKey != null)) {
            // found suitable join keys
            // add them to key list, ensuring that if there is a
            // non-equi join predicate, it appears at the end of the
            // key list; also mark the null filtering property
            addJoinKey(joinKeys.get(leftInput), leftKey, (rangeOp != null) && !rangeOp.isEmpty());
            addJoinKey(joinKeys.get(rightInput), rightKey, (rangeOp != null) && !rangeOp.isEmpty());
            if (filterNulls != null && kind == SqlKind.EQUALS) {
                // nulls are considered not matching for equality comparison
                // add the position of the most recently inserted key
                filterNulls.add(joinKeys.get(leftInput).size() - 1);
            }
            if (rangeOp != null && kind != SqlKind.EQUALS && kind != SqlKind.IS_DISTINCT_FROM) {
                if (reverse) {
                    kind = reverse(kind);
                }
                rangeOp.add(op(kind, call.getOperator()));
            }
            return;
        }
    // else fall through and add this condition as nonEqui condition
    }
    // The operator is not of RexCall type
    // So we fail. Fall through.
    // Add this condition to the list of non-equi-join conditions.
    nonEquiList.add(condition);
}

Example 47 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveJoinPushTransitivePredicatesRule method getTypeSafePred.

private RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
    RexNode typeSafeRex = rex;
    if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
        RexBuilder rb = cluster.getRexBuilder();
        List<RexNode> fixedPredElems = new ArrayList<RexNode>();
        RelDataType commonType = cluster.getTypeFactory().leastRestrictive(RexUtil.types(((RexCall) rex).getOperands()));
        for (RexNode rn : ((RexCall) rex).getOperands()) {
            fixedPredElems.add(rb.ensureType(commonType, rn, true));
        }
        typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
    }
    return typeSafeRex;
}

Example 48 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveJoinPushTransitivePredicatesRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    Join join = call.rel(0);
    RelOptPredicateList preds = RelMetadataQuery.instance().getPulledUpPredicates(join);
    HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
    assert registry != null;
    RexBuilder rB = join.getCluster().getRexBuilder();
    RelNode lChild = join.getLeft();
    RelNode rChild = join.getRight();
    Set<String> leftPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 0));
    List<RexNode> leftPreds = getValidPreds(join.getCluster(), lChild, leftPushedPredicates, preds.leftInferredPredicates, lChild.getRowType());
    Set<String> rightPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 1));
    List<RexNode> rightPreds = getValidPreds(join.getCluster(), rChild, rightPushedPredicates, preds.rightInferredPredicates, rChild.getRowType());
    RexNode newLeftPredicate = RexUtil.composeConjunction(rB, leftPreds, false);
    RexNode newRightPredicate = RexUtil.composeConjunction(rB, rightPreds, false);
    if (newLeftPredicate.isAlwaysTrue() && newRightPredicate.isAlwaysTrue()) {
        return;
    }
    if (!newLeftPredicate.isAlwaysTrue()) {
        RelNode curr = lChild;
        lChild = filterFactory.createFilter(lChild, newLeftPredicate);
        call.getPlanner().onCopy(curr, lChild);
    }
    if (!newRightPredicate.isAlwaysTrue()) {
        RelNode curr = rChild;
        rChild = filterFactory.createFilter(rChild, newRightPredicate);
        call.getPlanner().onCopy(curr, rChild);
    }
    RelNode newRel = join.copy(join.getTraitSet(), join.getCondition(), lChild, rChild, join.getJoinType(), join.isSemiJoinDone());
    call.getPlanner().onCopy(join, newRel);
    // Register information about pushed predicates
    registry.getPushedPredicates(newRel, 0).addAll(leftPushedPredicates);
    registry.getPushedPredicates(newRel, 1).addAll(rightPushedPredicates);
    call.transformTo(newRel);
}

Example 49 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveJoinToMultiJoinRule method mergeJoin.

// This method tries to merge the join with its left child. The left
// child should be a join for this to happen.
private static RelNode mergeJoin(HiveJoin join, RelNode left, RelNode right) {
    final RexBuilder rexBuilder = join.getCluster().getRexBuilder();
    // We check whether the join can be combined with any of its children
    final List<RelNode> newInputs = Lists.newArrayList();
    final List<RexNode> newJoinCondition = Lists.newArrayList();
    final List<Pair<Integer, Integer>> joinInputs = Lists.newArrayList();
    final List<JoinRelType> joinTypes = Lists.newArrayList();
    final List<RexNode> joinFilters = Lists.newArrayList();
    // Left child
    if (left instanceof HiveJoin || left instanceof HiveMultiJoin) {
        final RexNode leftCondition;
        final List<Pair<Integer, Integer>> leftJoinInputs;
        final List<JoinRelType> leftJoinTypes;
        final List<RexNode> leftJoinFilters;
        boolean combinable;
        if (left instanceof HiveJoin) {
            HiveJoin hj = (HiveJoin) left;
            leftCondition = hj.getCondition();
            leftJoinInputs = ImmutableList.of(Pair.of(0, 1));
            leftJoinTypes = ImmutableList.of(hj.getJoinType());
            leftJoinFilters = ImmutableList.of(hj.getJoinFilter());
            try {
                combinable = isCombinableJoin(join, hj);
            } catch (CalciteSemanticException e) {
                LOG.trace("Failed to merge join-join", e);
                combinable = false;
            }
        } else {
            HiveMultiJoin hmj = (HiveMultiJoin) left;
            leftCondition = hmj.getCondition();
            leftJoinInputs = hmj.getJoinInputs();
            leftJoinTypes = hmj.getJoinTypes();
            leftJoinFilters = hmj.getJoinFilters();
            try {
                combinable = isCombinableJoin(join, hmj);
            } catch (CalciteSemanticException e) {
                LOG.trace("Failed to merge join-multijoin", e);
                combinable = false;
            }
        }
        if (combinable) {
            newJoinCondition.add(leftCondition);
            for (int i = 0; i < leftJoinInputs.size(); i++) {
                joinInputs.add(leftJoinInputs.get(i));
                joinTypes.add(leftJoinTypes.get(i));
                joinFilters.add(leftJoinFilters.get(i));
            }
            newInputs.addAll(left.getInputs());
        } else {
            // The join operation in the child is not on the same keys
            return null;
        }
    } else {
        // The left child is not a join or multijoin operator
        return null;
    }
    final int numberLeftInputs = newInputs.size();
    // Right child
    newInputs.add(right);
    // If we cannot combine any of the children, we bail out
    newJoinCondition.add(join.getCondition());
    if (newJoinCondition.size() == 1) {
        return null;
    }
    final List<RelDataTypeField> systemFieldList = ImmutableList.of();
    List<List<RexNode>> joinKeyExprs = new ArrayList<List<RexNode>>();
    List<Integer> filterNulls = new ArrayList<Integer>();
    for (int i = 0; i < newInputs.size(); i++) {
        joinKeyExprs.add(new ArrayList<RexNode>());
    }
    RexNode filters;
    try {
        filters = HiveRelOptUtil.splitHiveJoinCondition(systemFieldList, newInputs, join.getCondition(), joinKeyExprs, filterNulls, null);
    } catch (CalciteSemanticException e) {
        LOG.trace("Failed to merge joins", e);
        return null;
    }
    ImmutableBitSet.Builder keysInInputsBuilder = ImmutableBitSet.builder();
    for (int i = 0; i < newInputs.size(); i++) {
        List<RexNode> partialCondition = joinKeyExprs.get(i);
        if (!partialCondition.isEmpty()) {
            keysInInputsBuilder.set(i);
        }
    }
    // If we cannot merge, we bail out
    ImmutableBitSet keysInInputs = keysInInputsBuilder.build();
    ImmutableBitSet leftReferencedInputs = keysInInputs.intersect(ImmutableBitSet.range(numberLeftInputs));
    ImmutableBitSet rightReferencedInputs = keysInInputs.intersect(ImmutableBitSet.range(numberLeftInputs, newInputs.size()));
    if (join.getJoinType() != JoinRelType.INNER && (leftReferencedInputs.cardinality() > 1 || rightReferencedInputs.cardinality() > 1)) {
        return null;
    }
    // Otherwise, we add to the join specs
    if (join.getJoinType() != JoinRelType.INNER) {
        int leftInput = keysInInputs.nextSetBit(0);
        int rightInput = keysInInputs.nextSetBit(numberLeftInputs);
        joinInputs.add(Pair.of(leftInput, rightInput));
        joinTypes.add(join.getJoinType());
        joinFilters.add(filters);
    } else {
        for (int i : leftReferencedInputs) {
            for (int j : rightReferencedInputs) {
                joinInputs.add(Pair.of(i, j));
                joinTypes.add(join.getJoinType());
                joinFilters.add(filters);
            }
        }
    }
    // We can now create a multijoin operator
    RexNode newCondition = RexUtil.flatten(rexBuilder, RexUtil.composeConjunction(rexBuilder, newJoinCondition, false));
    List<RelNode> newInputsArray = Lists.newArrayList(newInputs);
    JoinPredicateInfo joinPredInfo = null;
    try {
        joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(newInputsArray, systemFieldList, newCondition);
    } catch (CalciteSemanticException e) {
        throw new RuntimeException(e);
    }
    // If the number of joins < number of input tables-1, this is not a star join.
    if (joinPredInfo.getEquiJoinPredicateElements().size() < newInputs.size() - 1) {
        return null;
    }
    // Validate that the multi-join is a valid star join before returning it.
    for (int i = 0; i < newInputs.size(); i++) {
        List<RexNode> joinKeys = null;
        for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
            List<RexNode> currJoinKeys = joinPredInfo.getEquiJoinPredicateElements().get(j).getJoinExprs(i);
            if (currJoinKeys.isEmpty()) {
                continue;
            }
            if (joinKeys == null) {
                joinKeys = currJoinKeys;
            } else {
                // Bail out if this is the case.
                if (!joinKeys.containsAll(currJoinKeys) || !currJoinKeys.containsAll(joinKeys)) {
                    return null;
                }
            }
        }
    }
    return new HiveMultiJoin(join.getCluster(), newInputsArray, newCondition, join.getRowType(), joinInputs, joinTypes, joinFilters, joinPredInfo);
}

Example 50 with RexBuilder

use of org.apache.calcite.rex.RexBuilder in project hive by apache.

the class HiveProject method projectMapping.

/**
   * Creates a relational expression which projects the output fields of a
   * relational expression according to a partial mapping.
   *
   * <p>
   * A partial mapping is weaker than a permutation: every target has one
   * source, but a source may have 0, 1 or more than one targets. Usually the
   * result will have fewer fields than the source, unless some source fields
   * are projected multiple times.
   *
   * <p>
   * This method could optimize the result as {@link #permute} does, but does
   * not at present.
   *
   * @param rel
   *          Relational expression
   * @param mapping
   *          Mapping from source fields to target fields. The mapping type must
   *          obey the constraints {@link MappingType#isMandatorySource()} and
   *          {@link MappingType#isSingleSource()}, as does
   *          {@link MappingType#INVERSE_FUNCTION}.
   * @param fieldNames
   *          Field names; if null, or if a particular entry is null, the name
   *          of the permuted field is used
   * @return relational expression which projects a subset of the input fields
   * @throws CalciteSemanticException
   */
public static RelNode projectMapping(RelNode rel, Mapping mapping, List<String> fieldNames) throws CalciteSemanticException {
    assert mapping.getMappingType().isSingleSource();
    assert mapping.getMappingType().isMandatorySource();
    if (mapping.isIdentity()) {
        return rel;
    }
    final List<String> outputNameList = new ArrayList<String>();
    final List<RexNode> outputProjList = new ArrayList<RexNode>();
    final List<RelDataTypeField> fields = rel.getRowType().getFieldList();
    final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    for (int i = 0; i < mapping.getTargetCount(); i++) {
        int source = mapping.getSource(i);
        final RelDataTypeField sourceField = fields.get(source);
        outputNameList.add(((fieldNames == null) || (fieldNames.size() <= i) || (fieldNames.get(i) == null)) ? sourceField.getName() : fieldNames.get(i));
        outputProjList.add(rexBuilder.makeInputRef(rel, source));
    }
    return create(rel, outputProjList, outputNameList);
}