Examples with RelTableRef org.apache.calcite.rex.RexTableInputRef.RelTableRef used on opensource projects

Example 6 with RelTableRef

use of org.apache.calcite.rex.RexTableInputRef.RelTableRef in project hive by apache.

the class HiveRelOptUtil method getColumnOriginSet.

public static Pair<RelOptTable, List<Integer>> getColumnOriginSet(RelNode rel, ImmutableBitSet colSet) {
    RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
    RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    Map<RelTableRef, List<Integer>> tabToOriginColumns = new HashMap<>();
    for (int col : colSet) {
        final RexInputRef tempColRef = rexBuilder.makeInputRef(rel, col);
        Set<RexNode> columnOrigins = mq.getExpressionLineage(rel, tempColRef);
        if (null == columnOrigins || columnOrigins.isEmpty()) {
            // if even on
            return null;
        }
        // we have either one or multiple origins of the column, we need to make sure that all of the column
        for (RexNode orgCol : columnOrigins) {
            RexTableInputRef inputRef = extractTableInputRef(orgCol);
            if (inputRef == null) {
                return null;
            }
            List<Integer> cols = tabToOriginColumns.get(inputRef.getTableRef());
            if (cols == null) {
                cols = new ArrayList<>();
            }
            cols.add(inputRef.getIndex());
            tabToOriginColumns.put(inputRef.getTableRef(), cols);
        }
    }
    // ideally we should return all, in case one doesn't work we can fall back to another
    for (Entry<RelTableRef, List<Integer>> mapEntries : tabToOriginColumns.entrySet()) {
        RelTableRef tblRef = mapEntries.getKey();
        List<Integer> mapColList = mapEntries.getValue();
        if (mapColList.size() == colSet.cardinality()) {
            RelOptTable tbl = tblRef.getTable();
            return Pair.of(tbl, mapColList);
        }
    }
    return null;
}

Example 7 with RelTableRef

use of org.apache.calcite.rex.RexTableInputRef.RelTableRef in project hive by apache.

the class HiveRelOptUtil method isRewritablePKFKJoin.

public static RewritablePKFKJoinInfo isRewritablePKFKJoin(Join join, final RelNode fkInput, final RelNode nonFkInput, RelMetadataQuery mq) {
    final JoinRelType joinType = join.getJoinType();
    final RexNode cond = join.getCondition();
    Preconditions.checkArgument(fkInput == join.getLeft() || fkInput == join.getRight(), "Invalid input: " + fkInput);
    Preconditions.checkArgument(nonFkInput == join.getLeft() || nonFkInput == join.getRight(), "Invalid input: " + nonFkInput);
    final RewritablePKFKJoinInfo nonRewritable = RewritablePKFKJoinInfo.of(false, null);
    // https://issues.apache.org/jira/browse/HIVE-23906
    if (joinType == JoinRelType.ANTI) {
        return nonRewritable;
    }
    if (joinType != JoinRelType.INNER && !join.isSemiJoin()) {
        // If it is not an inner, we transform it as the metadata
        // providers for expressions do not pull information through
        // outer join (as it would not be correct)
        join = join.copy(join.getTraitSet(), cond, join.getLeft(), join.getRight(), JoinRelType.INNER, false);
    }
    // below altered the PK cardinality in any way
    if (HiveRelOptUtil.isRowFilteringPlan(mq, nonFkInput)) {
        return nonRewritable;
    }
    // 2) Check whether there is an FK relationship
    final Map<RexTableInputRef, RexNode> refToRex = new HashMap<>();
    final EquivalenceClasses ec = new EquivalenceClasses();
    for (RexNode conj : RelOptUtil.conjunctions(cond)) {
        if (!conj.isA(SqlKind.EQUALS)) {
            // Not an equality, we bail out
            return nonRewritable;
        }
        RexCall equiCond = (RexCall) conj;
        RexNode eqOp1 = equiCond.getOperands().get(0);
        Set<RexNode> eqOp1ExprsLineage = mq.getExpressionLineage(join, eqOp1);
        if (eqOp1ExprsLineage == null) {
            // Cannot be mapped, bail out
            return nonRewritable;
        }
        RexNode eqOp2 = equiCond.getOperands().get(1);
        Set<RexNode> eqOp2ExprsLineage = mq.getExpressionLineage(join, eqOp2);
        if (eqOp2ExprsLineage == null) {
            // Cannot be mapped, bail out
            return nonRewritable;
        }
        List<RexTableInputRef> eqOp2ExprsFiltered = null;
        for (RexNode eqOpExprLineage1 : eqOp1ExprsLineage) {
            RexTableInputRef inputRef1 = extractTableInputRef(eqOpExprLineage1);
            if (inputRef1 == null) {
                // Bail out as this condition could not be map into an input reference
                return nonRewritable;
            }
            refToRex.put(inputRef1, eqOp1);
            if (eqOp2ExprsFiltered == null) {
                // First iteration
                eqOp2ExprsFiltered = new ArrayList<>();
                for (RexNode eqOpExprLineage2 : eqOp2ExprsLineage) {
                    RexTableInputRef inputRef2 = extractTableInputRef(eqOpExprLineage2);
                    if (inputRef2 == null) {
                        // Bail out as this condition could not be map into an input reference
                        return nonRewritable;
                    }
                    // Add to list of expressions for follow-up iterations
                    eqOp2ExprsFiltered.add(inputRef2);
                    // Add to equivalence classes and backwards mapping
                    ec.addEquivalence(inputRef1, inputRef2);
                    refToRex.put(inputRef2, eqOp2);
                }
            } else {
                // Rest of iterations, only adding, no checking
                for (RexTableInputRef inputRef2 : eqOp2ExprsFiltered) {
                    ec.addEquivalence(inputRef1, inputRef2);
                }
            }
        }
    }
    if (ec.getEquivalenceClassesMap().isEmpty()) {
        // This may be a cartesian product, we bail out
        return nonRewritable;
    }
    // 3) Gather all tables from the FK side and the table from the
    // non-FK side
    final Set<RelTableRef> leftTables = mq.getTableReferences(join.getLeft());
    final Set<RelTableRef> rightTables = Sets.difference(mq.getTableReferences(join), mq.getTableReferences(join.getLeft()));
    final Set<RelTableRef> fkTables = join.getLeft() == fkInput ? leftTables : rightTables;
    final Set<RelTableRef> nonFkTables = join.getLeft() == fkInput ? rightTables : leftTables;
    assert nonFkTables.size() == 1;
    final RelTableRef nonFkTable = nonFkTables.iterator().next();
    final List<String> nonFkTableQName = nonFkTable.getQualifiedName();
    // 4) For each table, check whether there is a matching on the non-FK side.
    // If there is and it is the only condition, we are ready to transform
    boolean canBeRewritten = false;
    List<RexNode> nullableNodes = null;
    for (RelTableRef tRef : fkTables) {
        List<RelReferentialConstraint> constraints = tRef.getTable().getReferentialConstraints();
        for (RelReferentialConstraint constraint : constraints) {
            if (constraint.getTargetQualifiedName().equals(nonFkTableQName)) {
                nullableNodes = new ArrayList<>();
                EquivalenceClasses ecT = EquivalenceClasses.copy(ec);
                boolean allContained = true;
                for (int pos = 0; pos < constraint.getNumColumns(); pos++) {
                    int foreignKeyPos = constraint.getColumnPairs().get(pos).source;
                    RelDataType foreignKeyColumnType = tRef.getTable().getRowType().getFieldList().get(foreignKeyPos).getType();
                    RexTableInputRef foreignKeyColumnRef = RexTableInputRef.of(tRef, foreignKeyPos, foreignKeyColumnType);
                    int uniqueKeyPos = constraint.getColumnPairs().get(pos).target;
                    RexTableInputRef uniqueKeyColumnRef = RexTableInputRef.of(nonFkTable, uniqueKeyPos, nonFkTable.getTable().getRowType().getFieldList().get(uniqueKeyPos).getType());
                    if (ecT.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && ecT.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) {
                        if (foreignKeyColumnType.isNullable()) {
                            // TODO : Handle Anti Join. https://issues.apache.org/jira/browse/HIVE-23906
                            if (joinType == JoinRelType.INNER || join.isSemiJoin()) {
                                // If it is nullable and it is an INNER, we just need a IS NOT NULL filter
                                RexNode originalCondOp = refToRex.get(foreignKeyColumnRef);
                                assert originalCondOp != null;
                                nullableNodes.add(originalCondOp);
                            } else {
                                // If it is nullable and this is not an INNER, we cannot execute any transformation
                                allContained = false;
                                break;
                            }
                        }
                        // Remove this condition from eq classes as we have checked that it is present
                        // in the join condition
                        ecT.removeEquivalence(uniqueKeyColumnRef, foreignKeyColumnRef);
                    } else {
                        // No relationship, we cannot do anything
                        allContained = false;
                        break;
                    }
                }
                if (allContained && ecT.getEquivalenceClassesMap().isEmpty()) {
                    // We made it
                    canBeRewritten = true;
                    break;
                }
            }
        }
    }
    return RewritablePKFKJoinInfo.of(canBeRewritten, nullableNodes);
}

Example 8 with RelTableRef

use of org.apache.calcite.rex.RexTableInputRef.RelTableRef in project hive by apache.

the class HiveRelOptUtil method extractPKFKJoin.

/**
 * Returns a triple where first value represents whether we could extract a FK-PK join
 * or not, the second value is a pair with the column from left and right input that
 * are used for the FK-PK join, and the third value are the predicates that are not
 * part of the FK-PK condition. Currently we can only extract one FK-PK join.
 */
public static PKFKJoinInfo extractPKFKJoin(Join join, List<RexNode> joinFilters, boolean leftInputPotentialFK, RelMetadataQuery mq) {
    final List<RexNode> residualPreds = new ArrayList<>();
    final JoinRelType joinType = join.getJoinType();
    final RelNode fkInput = leftInputPotentialFK ? join.getLeft() : join.getRight();
    final PKFKJoinInfo cannotExtract = PKFKJoinInfo.of(false, null, null);
    if (joinType != JoinRelType.INNER) {
        // If it is not an inner, we transform it as the metadata
        // providers for expressions do not pull information through
        // outer join (as it would not be correct)
        join = join.copy(join.getTraitSet(), join.getCluster().getRexBuilder().makeLiteral(true), join.getLeft(), join.getRight(), JoinRelType.INNER, false);
    }
    // 1) Gather all tables from the FK side and the table from the
    // non-FK side
    final Set<RelTableRef> leftTables = mq.getTableReferences(join.getLeft());
    if (leftTables == null) {
        // Could not infer, bail out
        return cannotExtract;
    }
    final Set<RelTableRef> joinTables = mq.getTableReferences(join);
    if (joinTables == null) {
        // Could not infer, bail out
        return cannotExtract;
    }
    final Set<RelTableRef> rightTables = Sets.difference(joinTables, leftTables);
    final Set<RelTableRef> fkTables = join.getLeft() == fkInput ? leftTables : rightTables;
    final Set<RelTableRef> nonFkTables = join.getLeft() == fkInput ? rightTables : leftTables;
    // 2) Check whether there is a FK relationship
    Set<RexCall> candidatePredicates = new HashSet<>();
    EquivalenceClasses ec = new EquivalenceClasses();
    for (RexNode conj : joinFilters) {
        if (!conj.isA(SqlKind.EQUALS)) {
            // Not an equality, continue
            residualPreds.add(conj);
            continue;
        }
        RexCall equiCond = (RexCall) conj;
        RexNode eqOp1 = equiCond.getOperands().get(0);
        if (!RexUtil.isReferenceOrAccess(eqOp1, true)) {
            // Ignore
            residualPreds.add(conj);
            continue;
        }
        Set<RexNode> eqOp1ExprsLineage = mq.getExpressionLineage(join, eqOp1);
        if (eqOp1ExprsLineage == null) {
            // Cannot be mapped, continue
            residualPreds.add(conj);
            continue;
        }
        RexNode eqOp2 = equiCond.getOperands().get(1);
        if (!RexUtil.isReferenceOrAccess(eqOp2, true)) {
            // Ignore
            residualPreds.add(conj);
            continue;
        }
        Set<RexNode> eqOp2ExprsLineage = mq.getExpressionLineage(join, eqOp2);
        if (eqOp2ExprsLineage == null) {
            // Cannot be mapped, continue
            residualPreds.add(conj);
            continue;
        }
        List<RexTableInputRef> eqOp2ExprsFiltered = null;
        for (RexNode eqOpExprLineage1 : eqOp1ExprsLineage) {
            RexTableInputRef inputRef1 = extractTableInputRef(eqOpExprLineage1);
            if (inputRef1 == null) {
                // This condition could not be map into an input reference
                continue;
            }
            if (eqOp2ExprsFiltered == null) {
                // First iteration
                eqOp2ExprsFiltered = new ArrayList<>();
                for (RexNode eqOpExprLineage2 : eqOp2ExprsLineage) {
                    RexTableInputRef inputRef2 = extractTableInputRef(eqOpExprLineage2);
                    if (inputRef2 == null) {
                        // Bail out as this condition could not be map into an input reference
                        continue;
                    }
                    // Add to list of expressions for follow-up iterations
                    eqOp2ExprsFiltered.add(inputRef2);
                    // Add to equivalence classes and backwards mapping
                    ec.addEquivalence(inputRef1, inputRef2, equiCond);
                    candidatePredicates.add(equiCond);
                }
            } else {
                // Rest of iterations, only adding, no checking
                for (RexTableInputRef inputRef2 : eqOp2ExprsFiltered) {
                    ec.addEquivalence(inputRef1, inputRef2, equiCond);
                }
            }
        }
        if (!candidatePredicates.contains(conj)) {
            // We add it to residual already
            residualPreds.add(conj);
        }
    }
    if (ec.getEquivalenceClassesMap().isEmpty()) {
        // This may be a cartesian product, we bail out
        return cannotExtract;
    }
    // If there is and it is the only condition, we are ready to transform
    for (final RelTableRef nonFkTable : nonFkTables) {
        final List<String> nonFkTableQName = nonFkTable.getQualifiedName();
        for (RelTableRef tRef : fkTables) {
            List<RelReferentialConstraint> constraints = tRef.getTable().getReferentialConstraints();
            for (RelReferentialConstraint constraint : constraints) {
                if (constraint.getTargetQualifiedName().equals(nonFkTableQName)) {
                    EquivalenceClasses ecT = EquivalenceClasses.copy(ec);
                    Set<RexNode> removedOriginalPredicates = new HashSet<>();
                    ImmutableBitSet.Builder lBitSet = ImmutableBitSet.builder();
                    ImmutableBitSet.Builder rBitSet = ImmutableBitSet.builder();
                    boolean allContained = true;
                    for (int pos = 0; pos < constraint.getNumColumns(); pos++) {
                        int foreignKeyPos = constraint.getColumnPairs().get(pos).source;
                        RelDataType foreignKeyColumnType = tRef.getTable().getRowType().getFieldList().get(foreignKeyPos).getType();
                        RexTableInputRef foreignKeyColumnRef = RexTableInputRef.of(tRef, foreignKeyPos, foreignKeyColumnType);
                        int uniqueKeyPos = constraint.getColumnPairs().get(pos).target;
                        RexTableInputRef uniqueKeyColumnRef = RexTableInputRef.of(nonFkTable, uniqueKeyPos, nonFkTable.getTable().getRowType().getFieldList().get(uniqueKeyPos).getType());
                        if (ecT.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && ecT.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) {
                            // from the join inputs
                            for (RexCall originalPred : ecT.removeEquivalence(uniqueKeyColumnRef, foreignKeyColumnRef)) {
                                ImmutableBitSet leftCols = RelOptUtil.InputFinder.bits(originalPred.getOperands().get(0));
                                ImmutableBitSet rightCols = RelOptUtil.InputFinder.bits(originalPred.getOperands().get(1));
                                // Get length and flip column references if join condition specified in
                                // reverse order to join sources
                                int nFieldsLeft = join.getLeft().getRowType().getFieldList().size();
                                int nFieldsRight = join.getRight().getRowType().getFieldList().size();
                                int nSysFields = join.getSystemFieldList().size();
                                ImmutableBitSet rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
                                if (rightFieldsBitSet.contains(leftCols)) {
                                    ImmutableBitSet t = leftCols;
                                    leftCols = rightCols;
                                    rightCols = t;
                                }
                                lBitSet.set(leftCols.nextSetBit(0) - nSysFields);
                                rBitSet.set(rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft));
                                removedOriginalPredicates.add(originalPred);
                            }
                        } else {
                            // No relationship, we cannot do anything
                            allContained = false;
                            break;
                        }
                    }
                    if (allContained) {
                        // This is a PK-FK, reassign equivalence classes and remove conditions
                        // TODO: Support inference of multiple PK-FK relationships
                        // 4.1) Add to residual whatever is remaining
                        candidatePredicates.removeAll(removedOriginalPredicates);
                        residualPreds.addAll(candidatePredicates);
                        // 4.2) Return result
                        return PKFKJoinInfo.of(true, Pair.of(lBitSet.build(), rBitSet.build()), residualPreds);
                    }
                }
            }
        }
    }
    return cannotExtract;
}

Example 9 with RelTableRef

use of org.apache.calcite.rex.RexTableInputRef.RelTableRef in project calcite by apache.

the class RelMdTableReferences method getTableReferences.

/**
 * Table references from {@link Union}.
 *
 * <p>For Union operator, we might be able to extract multiple table
 * references.
 */
public Set<RelTableRef> getTableReferences(Union rel, RelMetadataQuery mq) {
    final Set<RelTableRef> result = new HashSet<>();
    // Infer column origin expressions for given references
    final Multimap<List<String>, RelTableRef> qualifiedNamesToRefs = HashMultimap.create();
    for (RelNode input : rel.getInputs()) {
        final Map<RelTableRef, RelTableRef> currentTablesMapping = new HashMap<>();
        for (RelTableRef tableRef : mq.getTableReferences(input)) {
            int shift = 0;
            Collection<RelTableRef> lRefs = qualifiedNamesToRefs.get(tableRef.getQualifiedName());
            if (lRefs != null) {
                shift = lRefs.size();
            }
            RelTableRef shiftTableRef = RelTableRef.of(tableRef.getTable(), shift + tableRef.getEntityNumber());
            assert !result.contains(shiftTableRef);
            result.add(shiftTableRef);
            currentTablesMapping.put(tableRef, shiftTableRef);
        }
        // Add to existing qualified names
        for (RelTableRef newRef : currentTablesMapping.values()) {
            qualifiedNamesToRefs.put(newRef.getQualifiedName(), newRef);
        }
    }
    // Return result
    return result;
}

Example 10 with RelTableRef

use of org.apache.calcite.rex.RexTableInputRef.RelTableRef in project calcite by apache.

the class RelMdTableReferences method getTableReferences.

/**
 * Table references from Join.
 */
public Set<RelTableRef> getTableReferences(Join rel, RelMetadataQuery mq) {
    final RelNode leftInput = rel.getLeft();
    final RelNode rightInput = rel.getRight();
    final Set<RelTableRef> result = new HashSet<>();
    // Gather table references, left input references remain unchanged
    final Multimap<List<String>, RelTableRef> leftQualifiedNamesToRefs = HashMultimap.create();
    for (RelTableRef leftRef : mq.getTableReferences(leftInput)) {
        assert !result.contains(leftRef);
        result.add(leftRef);
        leftQualifiedNamesToRefs.put(leftRef.getQualifiedName(), leftRef);
    }
    // updated if there are table names clashes with left input
    for (RelTableRef rightRef : mq.getTableReferences(rightInput)) {
        int shift = 0;
        Collection<RelTableRef> lRefs = leftQualifiedNamesToRefs.get(rightRef.getQualifiedName());
        if (lRefs != null) {
            shift = lRefs.size();
        }
        RelTableRef shiftTableRef = RelTableRef.of(rightRef.getTable(), shift + rightRef.getEntityNumber());
        assert !result.contains(shiftTableRef);
        result.add(shiftTableRef);
    }
    // Return result
    return result;
}