Examples with RexTableInputRef org.apache.calcite.rex.RexTableInputRef used on opensource projects

Example 16 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveRelFieldTrimmer method generateGroupSetIfCardinalitySame.

// Given a groupset this tries to find out if the cardinality of the grouping columns could have changed
// because if not and it consist of keys (unique + not null OR pk), we can safely remove rest of the columns
// if those are columns are not being used further up
private ImmutableBitSet generateGroupSetIfCardinalitySame(final Aggregate aggregate, final ImmutableBitSet originalGroupSet, final ImmutableBitSet fieldsUsed) {
    RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery();
    // map from backtracked table ref to list of gb keys and list of corresponding backtracked columns
    Map<RexTableInputRef.RelTableRef, List<Pair<Integer, Integer>>> mapGBKeysLineage = new HashMap<>();
    // map from table ref to list of columns (from gb keys) which are candidate to be removed
    Map<RexTableInputRef.RelTableRef, List<Integer>> candidateKeys = new HashMap<>();
    for (int key : originalGroupSet) {
        RexNode inputRef = rexBuilder.makeInputRef(aggregate.getInput(), key);
        Set<RexNode> exprLineage = mq.getExpressionLineage(aggregate.getInput(), inputRef);
        if (exprLineage != null && exprLineage.size() == 1) {
            RexNode expr = exprLineage.iterator().next();
            if (expr instanceof RexTableInputRef) {
                RexTableInputRef tblRef = (RexTableInputRef) expr;
                if (mapGBKeysLineage.containsKey(tblRef.getTableRef())) {
                    mapGBKeysLineage.get(tblRef.getTableRef()).add(Pair.of(tblRef.getIndex(), key));
                } else {
                    List<Pair<Integer, Integer>> newList = new ArrayList<>();
                    newList.add(Pair.of(tblRef.getIndex(), key));
                    mapGBKeysLineage.put(tblRef.getTableRef(), newList);
                }
            } else if (RexUtil.isDeterministic(expr)) {
                // even though we weren't able to backtrack this key it could still be candidate for removal
                // if rest of the columns contain pk/unique
                Set<RexTableInputRef.RelTableRef> tableRefs = RexUtil.gatherTableReferences(Lists.newArrayList(expr));
                if (tableRefs.size() == 1) {
                    RexTableInputRef.RelTableRef tblRef = tableRefs.iterator().next();
                    if (candidateKeys.containsKey(tblRef)) {
                        List<Integer> candidateGBKeys = candidateKeys.get(tblRef);
                        candidateGBKeys.add(key);
                    } else {
                        List<Integer> candidateGBKeys = new ArrayList<>();
                        candidateGBKeys.add(key);
                        candidateKeys.put(tblRef, candidateGBKeys);
                    }
                }
            }
        }
    }
    // we want to delete all columns in original GB set except the key
    ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
    for (Map.Entry<RexTableInputRef.RelTableRef, List<Pair<Integer, Integer>>> entry : mapGBKeysLineage.entrySet()) {
        RelOptHiveTable tbl = (RelOptHiveTable) entry.getKey().getTable();
        List<Pair<Integer, Integer>> gbKeyCols = entry.getValue();
        ImmutableBitSet.Builder btBuilder = ImmutableBitSet.builder();
        gbKeyCols.forEach(pair -> btBuilder.set(pair.left));
        ImmutableBitSet backtrackedGBSet = btBuilder.build();
        List<ImmutableBitSet> allKeys = tbl.getNonNullableKeys();
        ImmutableBitSet currentKey = null;
        for (ImmutableBitSet key : allKeys) {
            if (backtrackedGBSet.contains(key)) {
                // only if grouping sets consist of keys
                currentKey = key;
                break;
            }
        }
        if (currentKey == null || currentKey.isEmpty()) {
            continue;
        }
        // we have established that this gb set contains keys and it is safe to remove rest of the columns
        for (Pair<Integer, Integer> gbKeyColPair : gbKeyCols) {
            Integer backtrackedCol = gbKeyColPair.left;
            Integer orgCol = gbKeyColPair.right;
            if (!fieldsUsed.get(orgCol) && !currentKey.get(backtrackedCol)) {
                // this could could be removed
                builder.set(orgCol);
            }
        }
        // remove candidate keys if possible
        if (candidateKeys.containsKey(entry.getKey())) {
            List<Integer> candidateGbKeys = candidateKeys.get(entry.getKey());
            for (Integer keyToRemove : candidateGbKeys) {
                if (!fieldsUsed.get(keyToRemove)) {
                    builder.set(keyToRemove);
                }
            }
        }
    }
    ImmutableBitSet keysToRemove = builder.build();
    ImmutableBitSet newGroupSet = originalGroupSet.except(keysToRemove);
    assert (!newGroupSet.isEmpty());
    return newGroupSet;
}

Example 17 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveRelOptUtil method getColumnOriginSet.

public static Pair<RelOptTable, List<Integer>> getColumnOriginSet(RelNode rel, ImmutableBitSet colSet) {
    RelMetadataQuery mq = rel.getCluster().getMetadataQuery();
    RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    Map<RelTableRef, List<Integer>> tabToOriginColumns = new HashMap<>();
    for (int col : colSet) {
        final RexInputRef tempColRef = rexBuilder.makeInputRef(rel, col);
        Set<RexNode> columnOrigins = mq.getExpressionLineage(rel, tempColRef);
        if (null == columnOrigins || columnOrigins.isEmpty()) {
            // if even on
            return null;
        }
        // we have either one or multiple origins of the column, we need to make sure that all of the column
        for (RexNode orgCol : columnOrigins) {
            RexTableInputRef inputRef = extractTableInputRef(orgCol);
            if (inputRef == null) {
                return null;
            }
            List<Integer> cols = tabToOriginColumns.get(inputRef.getTableRef());
            if (cols == null) {
                cols = new ArrayList<>();
            }
            cols.add(inputRef.getIndex());
            tabToOriginColumns.put(inputRef.getTableRef(), cols);
        }
    }
    // ideally we should return all, in case one doesn't work we can fall back to another
    for (Entry<RelTableRef, List<Integer>> mapEntries : tabToOriginColumns.entrySet()) {
        RelTableRef tblRef = mapEntries.getKey();
        List<Integer> mapColList = mapEntries.getValue();
        if (mapColList.size() == colSet.cardinality()) {
            RelOptTable tbl = tblRef.getTable();
            return Pair.of(tbl, mapColList);
        }
    }
    return null;
}

Example 18 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveRelOptUtil method isRewritablePKFKJoin.

public static RewritablePKFKJoinInfo isRewritablePKFKJoin(Join join, final RelNode fkInput, final RelNode nonFkInput, RelMetadataQuery mq) {
    final JoinRelType joinType = join.getJoinType();
    final RexNode cond = join.getCondition();
    Preconditions.checkArgument(fkInput == join.getLeft() || fkInput == join.getRight(), "Invalid input: " + fkInput);
    Preconditions.checkArgument(nonFkInput == join.getLeft() || nonFkInput == join.getRight(), "Invalid input: " + nonFkInput);
    final RewritablePKFKJoinInfo nonRewritable = RewritablePKFKJoinInfo.of(false, null);
    // https://issues.apache.org/jira/browse/HIVE-23906
    if (joinType == JoinRelType.ANTI) {
        return nonRewritable;
    }
    if (joinType != JoinRelType.INNER && !join.isSemiJoin()) {
        // If it is not an inner, we transform it as the metadata
        // providers for expressions do not pull information through
        // outer join (as it would not be correct)
        join = join.copy(join.getTraitSet(), cond, join.getLeft(), join.getRight(), JoinRelType.INNER, false);
    }
    // below altered the PK cardinality in any way
    if (HiveRelOptUtil.isRowFilteringPlan(mq, nonFkInput)) {
        return nonRewritable;
    }
    // 2) Check whether there is an FK relationship
    final Map<RexTableInputRef, RexNode> refToRex = new HashMap<>();
    final EquivalenceClasses ec = new EquivalenceClasses();
    for (RexNode conj : RelOptUtil.conjunctions(cond)) {
        if (!conj.isA(SqlKind.EQUALS)) {
            // Not an equality, we bail out
            return nonRewritable;
        }
        RexCall equiCond = (RexCall) conj;
        RexNode eqOp1 = equiCond.getOperands().get(0);
        Set<RexNode> eqOp1ExprsLineage = mq.getExpressionLineage(join, eqOp1);
        if (eqOp1ExprsLineage == null) {
            // Cannot be mapped, bail out
            return nonRewritable;
        }
        RexNode eqOp2 = equiCond.getOperands().get(1);
        Set<RexNode> eqOp2ExprsLineage = mq.getExpressionLineage(join, eqOp2);
        if (eqOp2ExprsLineage == null) {
            // Cannot be mapped, bail out
            return nonRewritable;
        }
        List<RexTableInputRef> eqOp2ExprsFiltered = null;
        for (RexNode eqOpExprLineage1 : eqOp1ExprsLineage) {
            RexTableInputRef inputRef1 = extractTableInputRef(eqOpExprLineage1);
            if (inputRef1 == null) {
                // Bail out as this condition could not be map into an input reference
                return nonRewritable;
            }
            refToRex.put(inputRef1, eqOp1);
            if (eqOp2ExprsFiltered == null) {
                // First iteration
                eqOp2ExprsFiltered = new ArrayList<>();
                for (RexNode eqOpExprLineage2 : eqOp2ExprsLineage) {
                    RexTableInputRef inputRef2 = extractTableInputRef(eqOpExprLineage2);
                    if (inputRef2 == null) {
                        // Bail out as this condition could not be map into an input reference
                        return nonRewritable;
                    }
                    // Add to list of expressions for follow-up iterations
                    eqOp2ExprsFiltered.add(inputRef2);
                    // Add to equivalence classes and backwards mapping
                    ec.addEquivalence(inputRef1, inputRef2);
                    refToRex.put(inputRef2, eqOp2);
                }
            } else {
                // Rest of iterations, only adding, no checking
                for (RexTableInputRef inputRef2 : eqOp2ExprsFiltered) {
                    ec.addEquivalence(inputRef1, inputRef2);
                }
            }
        }
    }
    if (ec.getEquivalenceClassesMap().isEmpty()) {
        // This may be a cartesian product, we bail out
        return nonRewritable;
    }
    // 3) Gather all tables from the FK side and the table from the
    // non-FK side
    final Set<RelTableRef> leftTables = mq.getTableReferences(join.getLeft());
    final Set<RelTableRef> rightTables = Sets.difference(mq.getTableReferences(join), mq.getTableReferences(join.getLeft()));
    final Set<RelTableRef> fkTables = join.getLeft() == fkInput ? leftTables : rightTables;
    final Set<RelTableRef> nonFkTables = join.getLeft() == fkInput ? rightTables : leftTables;
    assert nonFkTables.size() == 1;
    final RelTableRef nonFkTable = nonFkTables.iterator().next();
    final List<String> nonFkTableQName = nonFkTable.getQualifiedName();
    // 4) For each table, check whether there is a matching on the non-FK side.
    // If there is and it is the only condition, we are ready to transform
    boolean canBeRewritten = false;
    List<RexNode> nullableNodes = null;
    for (RelTableRef tRef : fkTables) {
        List<RelReferentialConstraint> constraints = tRef.getTable().getReferentialConstraints();
        for (RelReferentialConstraint constraint : constraints) {
            if (constraint.getTargetQualifiedName().equals(nonFkTableQName)) {
                nullableNodes = new ArrayList<>();
                EquivalenceClasses ecT = EquivalenceClasses.copy(ec);
                boolean allContained = true;
                for (int pos = 0; pos < constraint.getNumColumns(); pos++) {
                    int foreignKeyPos = constraint.getColumnPairs().get(pos).source;
                    RelDataType foreignKeyColumnType = tRef.getTable().getRowType().getFieldList().get(foreignKeyPos).getType();
                    RexTableInputRef foreignKeyColumnRef = RexTableInputRef.of(tRef, foreignKeyPos, foreignKeyColumnType);
                    int uniqueKeyPos = constraint.getColumnPairs().get(pos).target;
                    RexTableInputRef uniqueKeyColumnRef = RexTableInputRef.of(nonFkTable, uniqueKeyPos, nonFkTable.getTable().getRowType().getFieldList().get(uniqueKeyPos).getType());
                    if (ecT.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && ecT.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) {
                        if (foreignKeyColumnType.isNullable()) {
                            // TODO : Handle Anti Join. https://issues.apache.org/jira/browse/HIVE-23906
                            if (joinType == JoinRelType.INNER || join.isSemiJoin()) {
                                // If it is nullable and it is an INNER, we just need a IS NOT NULL filter
                                RexNode originalCondOp = refToRex.get(foreignKeyColumnRef);
                                assert originalCondOp != null;
                                nullableNodes.add(originalCondOp);
                            } else {
                                // If it is nullable and this is not an INNER, we cannot execute any transformation
                                allContained = false;
                                break;
                            }
                        }
                        // Remove this condition from eq classes as we have checked that it is present
                        // in the join condition
                        ecT.removeEquivalence(uniqueKeyColumnRef, foreignKeyColumnRef);
                    } else {
                        // No relationship, we cannot do anything
                        allContained = false;
                        break;
                    }
                }
                if (allContained && ecT.getEquivalenceClassesMap().isEmpty()) {
                    // We made it
                    canBeRewritten = true;
                    break;
                }
            }
        }
    }
    return RewritablePKFKJoinInfo.of(canBeRewritten, nullableNodes);
}

Example 19 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveRelOptUtil method extractPKFKJoin.

/**
 * Returns a triple where first value represents whether we could extract a FK-PK join
 * or not, the second value is a pair with the column from left and right input that
 * are used for the FK-PK join, and the third value are the predicates that are not
 * part of the FK-PK condition. Currently we can only extract one FK-PK join.
 */
public static PKFKJoinInfo extractPKFKJoin(Join join, List<RexNode> joinFilters, boolean leftInputPotentialFK, RelMetadataQuery mq) {
    final List<RexNode> residualPreds = new ArrayList<>();
    final JoinRelType joinType = join.getJoinType();
    final RelNode fkInput = leftInputPotentialFK ? join.getLeft() : join.getRight();
    final PKFKJoinInfo cannotExtract = PKFKJoinInfo.of(false, null, null);
    if (joinType != JoinRelType.INNER) {
        // If it is not an inner, we transform it as the metadata
        // providers for expressions do not pull information through
        // outer join (as it would not be correct)
        join = join.copy(join.getTraitSet(), join.getCluster().getRexBuilder().makeLiteral(true), join.getLeft(), join.getRight(), JoinRelType.INNER, false);
    }
    // 1) Gather all tables from the FK side and the table from the
    // non-FK side
    final Set<RelTableRef> leftTables = mq.getTableReferences(join.getLeft());
    if (leftTables == null) {
        // Could not infer, bail out
        return cannotExtract;
    }
    final Set<RelTableRef> joinTables = mq.getTableReferences(join);
    if (joinTables == null) {
        // Could not infer, bail out
        return cannotExtract;
    }
    final Set<RelTableRef> rightTables = Sets.difference(joinTables, leftTables);
    final Set<RelTableRef> fkTables = join.getLeft() == fkInput ? leftTables : rightTables;
    final Set<RelTableRef> nonFkTables = join.getLeft() == fkInput ? rightTables : leftTables;
    // 2) Check whether there is a FK relationship
    Set<RexCall> candidatePredicates = new HashSet<>();
    EquivalenceClasses ec = new EquivalenceClasses();
    for (RexNode conj : joinFilters) {
        if (!conj.isA(SqlKind.EQUALS)) {
            // Not an equality, continue
            residualPreds.add(conj);
            continue;
        }
        RexCall equiCond = (RexCall) conj;
        RexNode eqOp1 = equiCond.getOperands().get(0);
        if (!RexUtil.isReferenceOrAccess(eqOp1, true)) {
            // Ignore
            residualPreds.add(conj);
            continue;
        }
        Set<RexNode> eqOp1ExprsLineage = mq.getExpressionLineage(join, eqOp1);
        if (eqOp1ExprsLineage == null) {
            // Cannot be mapped, continue
            residualPreds.add(conj);
            continue;
        }
        RexNode eqOp2 = equiCond.getOperands().get(1);
        if (!RexUtil.isReferenceOrAccess(eqOp2, true)) {
            // Ignore
            residualPreds.add(conj);
            continue;
        }
        Set<RexNode> eqOp2ExprsLineage = mq.getExpressionLineage(join, eqOp2);
        if (eqOp2ExprsLineage == null) {
            // Cannot be mapped, continue
            residualPreds.add(conj);
            continue;
        }
        List<RexTableInputRef> eqOp2ExprsFiltered = null;
        for (RexNode eqOpExprLineage1 : eqOp1ExprsLineage) {
            RexTableInputRef inputRef1 = extractTableInputRef(eqOpExprLineage1);
            if (inputRef1 == null) {
                // This condition could not be map into an input reference
                continue;
            }
            if (eqOp2ExprsFiltered == null) {
                // First iteration
                eqOp2ExprsFiltered = new ArrayList<>();
                for (RexNode eqOpExprLineage2 : eqOp2ExprsLineage) {
                    RexTableInputRef inputRef2 = extractTableInputRef(eqOpExprLineage2);
                    if (inputRef2 == null) {
                        // Bail out as this condition could not be map into an input reference
                        continue;
                    }
                    // Add to list of expressions for follow-up iterations
                    eqOp2ExprsFiltered.add(inputRef2);
                    // Add to equivalence classes and backwards mapping
                    ec.addEquivalence(inputRef1, inputRef2, equiCond);
                    candidatePredicates.add(equiCond);
                }
            } else {
                // Rest of iterations, only adding, no checking
                for (RexTableInputRef inputRef2 : eqOp2ExprsFiltered) {
                    ec.addEquivalence(inputRef1, inputRef2, equiCond);
                }
            }
        }
        if (!candidatePredicates.contains(conj)) {
            // We add it to residual already
            residualPreds.add(conj);
        }
    }
    if (ec.getEquivalenceClassesMap().isEmpty()) {
        // This may be a cartesian product, we bail out
        return cannotExtract;
    }
    // If there is and it is the only condition, we are ready to transform
    for (final RelTableRef nonFkTable : nonFkTables) {
        final List<String> nonFkTableQName = nonFkTable.getQualifiedName();
        for (RelTableRef tRef : fkTables) {
            List<RelReferentialConstraint> constraints = tRef.getTable().getReferentialConstraints();
            for (RelReferentialConstraint constraint : constraints) {
                if (constraint.getTargetQualifiedName().equals(nonFkTableQName)) {
                    EquivalenceClasses ecT = EquivalenceClasses.copy(ec);
                    Set<RexNode> removedOriginalPredicates = new HashSet<>();
                    ImmutableBitSet.Builder lBitSet = ImmutableBitSet.builder();
                    ImmutableBitSet.Builder rBitSet = ImmutableBitSet.builder();
                    boolean allContained = true;
                    for (int pos = 0; pos < constraint.getNumColumns(); pos++) {
                        int foreignKeyPos = constraint.getColumnPairs().get(pos).source;
                        RelDataType foreignKeyColumnType = tRef.getTable().getRowType().getFieldList().get(foreignKeyPos).getType();
                        RexTableInputRef foreignKeyColumnRef = RexTableInputRef.of(tRef, foreignKeyPos, foreignKeyColumnType);
                        int uniqueKeyPos = constraint.getColumnPairs().get(pos).target;
                        RexTableInputRef uniqueKeyColumnRef = RexTableInputRef.of(nonFkTable, uniqueKeyPos, nonFkTable.getTable().getRowType().getFieldList().get(uniqueKeyPos).getType());
                        if (ecT.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && ecT.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) {
                            // from the join inputs
                            for (RexCall originalPred : ecT.removeEquivalence(uniqueKeyColumnRef, foreignKeyColumnRef)) {
                                ImmutableBitSet leftCols = RelOptUtil.InputFinder.bits(originalPred.getOperands().get(0));
                                ImmutableBitSet rightCols = RelOptUtil.InputFinder.bits(originalPred.getOperands().get(1));
                                // Get length and flip column references if join condition specified in
                                // reverse order to join sources
                                int nFieldsLeft = join.getLeft().getRowType().getFieldList().size();
                                int nFieldsRight = join.getRight().getRowType().getFieldList().size();
                                int nSysFields = join.getSystemFieldList().size();
                                ImmutableBitSet rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
                                if (rightFieldsBitSet.contains(leftCols)) {
                                    ImmutableBitSet t = leftCols;
                                    leftCols = rightCols;
                                    rightCols = t;
                                }
                                lBitSet.set(leftCols.nextSetBit(0) - nSysFields);
                                rBitSet.set(rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft));
                                removedOriginalPredicates.add(originalPred);
                            }
                        } else {
                            // No relationship, we cannot do anything
                            allContained = false;
                            break;
                        }
                    }
                    if (allContained) {
                        // This is a PK-FK, reassign equivalence classes and remove conditions
                        // TODO: Support inference of multiple PK-FK relationships
                        // 4.1) Add to residual whatever is remaining
                        candidatePredicates.removeAll(removedOriginalPredicates);
                        residualPreds.addAll(candidatePredicates);
                        // 4.2) Return result
                        return PKFKJoinInfo.of(true, Pair.of(lBitSet.build(), rBitSet.build()), residualPreds);
                    }
                }
            }
        }
    }
    return cannotExtract;
}

Example 20 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.

the class RelMetadataTest method testExpressionLineageInnerJoinRight.

@Test
public void testExpressionLineageInnerJoinRight() {
    // ename is column 0 in catalog.sales.bonus
    final RelNode rel = convertSql("select bonus.ename from emp join bonus using (ename)");
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RexNode ref = RexInputRef.of(0, rel.getRowType().getFieldList());
    final Set<RexNode> r = mq.getExpressionLineage(rel, ref);
    assertThat(r.size(), is(1));
    final RexTableInputRef result = (RexTableInputRef) r.iterator().next();
    assertTrue(result.getQualifiedName().equals(ImmutableList.of("CATALOG", "SALES", "BONUS")));
    assertThat(result.getIndex(), is(0));
}