Examples with Pair org.apache.calcite.util.Pair used on opensource projects

Example 81 with Pair

use of org.apache.calcite.util.Pair in project flink by apache.

the class HiveParserCalcitePlanner method genDistSortBy.

// Generate plan for sort by, cluster by and distribute by. This is basically same as generating
// order by plan.
// Should refactor to combine them.
private Pair<RelNode, RelNode> genDistSortBy(HiveParserQB qb, RelNode srcRel, boolean outermostOB) throws SemanticException {
    RelNode res = null;
    RelNode originalInput = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    String destClause = qbp.getClauseNames().iterator().next();
    HiveParserASTNode sortAST = qbp.getSortByForClause(destClause);
    HiveParserASTNode distAST = qbp.getDistributeByForClause(destClause);
    HiveParserASTNode clusterAST = qbp.getClusterByForClause(destClause);
    if (sortAST != null || distAST != null || clusterAST != null) {
        List<RexNode> virtualCols = new ArrayList<>();
        List<Pair<HiveParserASTNode, TypeInfo>> vcASTAndType = new ArrayList<>();
        List<RelFieldCollation> fieldCollations = new ArrayList<>();
        List<Integer> distKeys = new ArrayList<>();
        HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
        HiveParserRexNodeConverter converter = new HiveParserRexNodeConverter(cluster, srcRel.getRowType(), relToHiveColNameCalcitePosMap.get(srcRel), 0, false, funcConverter);
        int numSrcFields = srcRel.getRowType().getFieldCount();
        // handle cluster by
        if (clusterAST != null) {
            if (sortAST != null) {
                throw new SemanticException("Cannot have both CLUSTER BY and SORT BY");
            }
            if (distAST != null) {
                throw new SemanticException("Cannot have both CLUSTER BY and DISTRIBUTE BY");
            }
            for (Node node : clusterAST.getChildren()) {
                HiveParserASTNode childAST = (HiveParserASTNode) node;
                Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(childAST, inputRR);
                ExprNodeDesc childNodeDesc = astToExprNodeDesc.get(childAST);
                if (childNodeDesc == null) {
                    throw new SemanticException("Invalid CLUSTER BY expression: " + childAST.toString());
                }
                RexNode childRexNode = converter.convert(childNodeDesc).accept(funcConverter);
                int fieldIndex;
                if (childRexNode instanceof RexInputRef) {
                    fieldIndex = ((RexInputRef) childRexNode).getIndex();
                } else {
                    fieldIndex = numSrcFields + virtualCols.size();
                    virtualCols.add(childRexNode);
                    vcASTAndType.add(new Pair<>(childAST, childNodeDesc.getTypeInfo()));
                }
                // cluster by doesn't support specifying ASC/DESC or NULLS FIRST/LAST, so use
                // default values
                fieldCollations.add(new RelFieldCollation(fieldIndex, RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST));
                distKeys.add(fieldIndex);
            }
        } else {
            // handle sort by
            if (sortAST != null) {
                for (Node node : sortAST.getChildren()) {
                    HiveParserASTNode childAST = (HiveParserASTNode) node;
                    HiveParserASTNode nullOrderAST = (HiveParserASTNode) childAST.getChild(0);
                    HiveParserASTNode fieldAST = (HiveParserASTNode) nullOrderAST.getChild(0);
                    Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(fieldAST, inputRR);
                    ExprNodeDesc fieldNodeDesc = astToExprNodeDesc.get(fieldAST);
                    if (fieldNodeDesc == null) {
                        throw new SemanticException("Invalid sort by expression: " + fieldAST.toString());
                    }
                    RexNode childRexNode = converter.convert(fieldNodeDesc).accept(funcConverter);
                    int fieldIndex;
                    if (childRexNode instanceof RexInputRef) {
                        fieldIndex = ((RexInputRef) childRexNode).getIndex();
                    } else {
                        fieldIndex = numSrcFields + virtualCols.size();
                        virtualCols.add(childRexNode);
                        vcASTAndType.add(new Pair<>(childAST, fieldNodeDesc.getTypeInfo()));
                    }
                    RelFieldCollation.Direction direction = RelFieldCollation.Direction.DESCENDING;
                    if (childAST.getType() == HiveASTParser.TOK_TABSORTCOLNAMEASC) {
                        direction = RelFieldCollation.Direction.ASCENDING;
                    }
                    RelFieldCollation.NullDirection nullOrder;
                    if (nullOrderAST.getType() == HiveASTParser.TOK_NULLS_FIRST) {
                        nullOrder = RelFieldCollation.NullDirection.FIRST;
                    } else if (nullOrderAST.getType() == HiveASTParser.TOK_NULLS_LAST) {
                        nullOrder = RelFieldCollation.NullDirection.LAST;
                    } else {
                        throw new SemanticException("Unexpected null ordering option: " + nullOrderAST.getType());
                    }
                    fieldCollations.add(new RelFieldCollation(fieldIndex, direction, nullOrder));
                }
            }
            // handle distribute by
            if (distAST != null) {
                for (Node node : distAST.getChildren()) {
                    HiveParserASTNode childAST = (HiveParserASTNode) node;
                    Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(childAST, inputRR);
                    ExprNodeDesc childNodeDesc = astToExprNodeDesc.get(childAST);
                    if (childNodeDesc == null) {
                        throw new SemanticException("Invalid DISTRIBUTE BY expression: " + childAST.toString());
                    }
                    RexNode childRexNode = converter.convert(childNodeDesc).accept(funcConverter);
                    int fieldIndex;
                    if (childRexNode instanceof RexInputRef) {
                        fieldIndex = ((RexInputRef) childRexNode).getIndex();
                    } else {
                        fieldIndex = numSrcFields + virtualCols.size();
                        virtualCols.add(childRexNode);
                        vcASTAndType.add(new Pair<>(childAST, childNodeDesc.getTypeInfo()));
                    }
                    distKeys.add(fieldIndex);
                }
            }
        }
        Preconditions.checkState(!fieldCollations.isEmpty() || !distKeys.isEmpty(), "Both field collations and dist keys are empty");
        // add child SEL if needed
        RelNode realInput = srcRel;
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        if (!virtualCols.isEmpty()) {
            List<RexNode> originalInputRefs = srcRel.getRowType().getFieldList().stream().map(input -> new RexInputRef(input.getIndex(), input.getType())).collect(Collectors.toList());
            HiveParserRowResolver addedProjectRR = new HiveParserRowResolver();
            if (!HiveParserRowResolver.add(addedProjectRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
            int vColPos = inputRR.getRowSchema().getSignature().size();
            for (Pair<HiveParserASTNode, TypeInfo> astTypePair : vcASTAndType) {
                addedProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo(getColumnInternalName(vColPos), astTypePair.getValue(), null, false));
                vColPos++;
            }
            realInput = genSelectRelNode(CompositeList.of(originalInputRefs, virtualCols), addedProjectRR, srcRel);
            if (outermostOB) {
                if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            } else {
                if (!HiveParserRowResolver.add(outputRR, addedProjectRR)) {
                    throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
                }
            }
            originalInput = srcRel;
        } else {
            if (!HiveParserRowResolver.add(outputRR, inputRR)) {
                throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
            }
        }
        // create rel node
        RelTraitSet traitSet = cluster.traitSet();
        RelCollation canonizedCollation = traitSet.canonize(RelCollationImpl.of(fieldCollations));
        res = LogicalDistribution.create(realInput, canonizedCollation, distKeys);
        Map<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(outputRR);
        relToRowResolver.put(res, outputRR);
        relToHiveColNameCalcitePosMap.put(res, hiveColNameCalcitePosMap);
    }
    return (new Pair<>(res, originalInput));
}

Example 82 with Pair

use of org.apache.calcite.util.Pair in project flink by apache.

the class HiveParserCalcitePlanner method genLogicalPlan.

private RelNode genLogicalPlan(HiveParserQB qb, boolean outerMostQB, Map<String, Integer> outerNameToPosMap, HiveParserRowResolver outerRR) throws SemanticException {
    RelNode res;
    // First generate all the opInfos for the elements in the from clause
    Map<String, RelNode> aliasToRel = new HashMap<>();
    // 0. Check if we can handle the SubQuery;
    // canHandleQbForCbo returns null if the query can be handled.
    String reason = HiveParserUtils.canHandleQbForCbo(semanticAnalyzer.getQueryProperties());
    if (reason != null) {
        String msg = "CBO can not handle Sub Query" + " because it: " + reason;
        throw new SemanticException(msg);
    }
    // 1.1. Recurse over the subqueries to fill the subquery part of the plan
    for (String subqAlias : qb.getSubqAliases()) {
        HiveParserQBExpr qbexpr = qb.getSubqForAlias(subqAlias);
        RelNode relNode = genLogicalPlan(qbexpr);
        aliasToRel.put(subqAlias, relNode);
        if (qb.getViewToTabSchema().containsKey(subqAlias)) {
            if (!(relNode instanceof Project)) {
                throw new SemanticException("View " + subqAlias + " is corresponding to " + relNode.toString() + ", rather than a Project.");
            }
        }
    }
    // 1.2 Recurse over all the source tables
    for (String tableAlias : qb.getTabAliases()) {
        RelNode op = genTableLogicalPlan(tableAlias, qb);
        aliasToRel.put(tableAlias, op);
    }
    if (aliasToRel.isEmpty()) {
        RelNode dummySrc = LogicalValues.createOneRow(cluster);
        aliasToRel.put(HiveParserSemanticAnalyzer.DUMMY_TABLE, dummySrc);
        HiveParserRowResolver dummyRR = new HiveParserRowResolver();
        dummyRR.put(HiveParserSemanticAnalyzer.DUMMY_TABLE, "dummy_col", new ColumnInfo(getColumnInternalName(0), TypeInfoFactory.intTypeInfo, HiveParserSemanticAnalyzer.DUMMY_TABLE, false));
        relToRowResolver.put(dummySrc, dummyRR);
        relToHiveColNameCalcitePosMap.put(dummySrc, buildHiveToCalciteColumnMap(dummyRR));
    }
    if (!qb.getParseInfo().getAliasToLateralViews().isEmpty()) {
        // process lateral views
        res = genLateralViewPlan(qb, aliasToRel);
    } else if (qb.getParseInfo().getJoinExpr() != null) {
        // 1.3 process join
        res = genJoinLogicalPlan(qb.getParseInfo().getJoinExpr(), aliasToRel);
    } else {
        // If no join then there should only be either 1 TS or 1 SubQuery
        res = aliasToRel.values().iterator().next();
    }
    // 2. Build Rel for where Clause
    RelNode filterRel = genFilterLogicalPlan(qb, res, outerNameToPosMap, outerRR);
    res = (filterRel == null) ? res : filterRel;
    RelNode starSrcRel = res;
    // 3. Build Rel for GB Clause
    RelNode gbRel = genGBLogicalPlan(qb, res);
    res = gbRel == null ? res : gbRel;
    // 4. Build Rel for GB Having Clause
    RelNode gbHavingRel = genGBHavingLogicalPlan(qb, res);
    res = gbHavingRel == null ? res : gbHavingRel;
    // 5. Build Rel for Select Clause
    RelNode selectRel = genSelectLogicalPlan(qb, res, starSrcRel, outerNameToPosMap, outerRR);
    res = selectRel == null ? res : selectRel;
    // 6. Build Rel for OB Clause
    Pair<Sort, RelNode> obAndTopProj = genOBLogicalPlan(qb, res, outerMostQB);
    Sort orderRel = obAndTopProj.getKey();
    RelNode topConstrainingProjRel = obAndTopProj.getValue();
    res = orderRel == null ? res : orderRel;
    // Build Rel for SortBy/ClusterBy/DistributeBy. It can happen only if we don't have OrderBy.
    if (orderRel == null) {
        Pair<RelNode, RelNode> distAndTopProj = genDistSortBy(qb, res, outerMostQB);
        RelNode distRel = distAndTopProj.getKey();
        topConstrainingProjRel = distAndTopProj.getValue();
        res = distRel == null ? res : distRel;
    }
    // 7. Build Rel for Limit Clause
    Sort limitRel = genLimitLogicalPlan(qb, res);
    if (limitRel != null) {
        if (orderRel != null) {
            // merge limit into the order-by node
            HiveParserRowResolver orderRR = relToRowResolver.remove(orderRel);
            Map<String, Integer> orderColNameToPos = relToHiveColNameCalcitePosMap.remove(orderRel);
            res = LogicalSort.create(orderRel.getInput(), orderRel.collation, limitRel.offset, limitRel.fetch);
            relToRowResolver.put(res, orderRR);
            relToHiveColNameCalcitePosMap.put(res, orderColNameToPos);
            relToRowResolver.remove(limitRel);
            relToHiveColNameCalcitePosMap.remove(limitRel);
        } else {
            res = limitRel;
        }
    }
    // 8. Introduce top constraining select if needed.
    if (topConstrainingProjRel != null) {
        List<RexNode> originalInputRefs = topConstrainingProjRel.getRowType().getFieldList().stream().map(input -> new RexInputRef(input.getIndex(), input.getType())).collect(Collectors.toList());
        HiveParserRowResolver topConstrainingProjRR = new HiveParserRowResolver();
        if (!HiveParserRowResolver.add(topConstrainingProjRR, relToRowResolver.get(topConstrainingProjRel))) {
            LOG.warn("Duplicates detected when adding columns to RR: see previous message");
        }
        res = genSelectRelNode(originalInputRefs, topConstrainingProjRR, res);
    }
    // TODO: cleanup this
    if (qb.getParseInfo().getAlias() != null) {
        HiveParserRowResolver rr = relToRowResolver.get(res);
        HiveParserRowResolver newRR = new HiveParserRowResolver();
        String alias = qb.getParseInfo().getAlias();
        for (ColumnInfo colInfo : rr.getColumnInfos()) {
            String name = colInfo.getInternalName();
            String[] tmp = rr.reverseLookup(name);
            if ("".equals(tmp[0]) || tmp[1] == null) {
                // ast expression is not a valid column name for table
                tmp[1] = colInfo.getInternalName();
            }
            ColumnInfo newColInfo = new ColumnInfo(colInfo);
            newColInfo.setTabAlias(alias);
            newRR.put(alias, tmp[1], newColInfo);
        }
        relToRowResolver.put(res, newRR);
        relToHiveColNameCalcitePosMap.put(res, buildHiveToCalciteColumnMap(newRR));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Created Plan for Query Block " + qb.getId());
    }
    semanticAnalyzer.setQB(qb);
    return res;
}

Example 83 with Pair

use of org.apache.calcite.util.Pair in project calcite by apache.

the class RelOptUtil method pushDownJoinConditions.

/**
 * Pushes down expressions in "equal" join condition.
 *
 * <p>For example, given
 * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above
 * "emp" that computes the expression
 * "emp.deptno + 1". The resulting join condition is a simple combination
 * of AND, equals, and input fields, plus the remaining non-equal conditions.
 *
 * @param originalJoin Join whose condition is to be pushed down
 * @param relBuilder Factory to create project operator
 */
public static RelNode pushDownJoinConditions(Join originalJoin, RelBuilder relBuilder) {
    RexNode joinCond = originalJoin.getCondition();
    final JoinRelType joinType = originalJoin.getJoinType();
    final List<RexNode> extraLeftExprs = new ArrayList<>();
    final List<RexNode> extraRightExprs = new ArrayList<>();
    final int leftCount = originalJoin.getLeft().getRowType().getFieldCount();
    final int rightCount = originalJoin.getRight().getRowType().getFieldCount();
    // yet.
    if (!containsGet(joinCond) && RexUtil.SubQueryFinder.find(joinCond) == null) {
        joinCond = pushDownEqualJoinConditions(joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs);
    }
    relBuilder.push(originalJoin.getLeft());
    if (!extraLeftExprs.isEmpty()) {
        final List<RelDataTypeField> fields = relBuilder.peek().getRowType().getFieldList();
        final List<Pair<RexNode, String>> pairs = new AbstractList<Pair<RexNode, String>>() {

            public int size() {
                return leftCount + extraLeftExprs.size();
            }

            public Pair<RexNode, String> get(int index) {
                if (index < leftCount) {
                    RelDataTypeField field = fields.get(index);
                    return Pair.<RexNode, String>of(new RexInputRef(index, field.getType()), field.getName());
                } else {
                    return Pair.of(extraLeftExprs.get(index - leftCount), null);
                }
            }
        };
        relBuilder.project(Pair.left(pairs), Pair.right(pairs));
    }
    relBuilder.push(originalJoin.getRight());
    if (!extraRightExprs.isEmpty()) {
        final List<RelDataTypeField> fields = relBuilder.peek().getRowType().getFieldList();
        final int newLeftCount = leftCount + extraLeftExprs.size();
        final List<Pair<RexNode, String>> pairs = new AbstractList<Pair<RexNode, String>>() {

            public int size() {
                return rightCount + extraRightExprs.size();
            }

            public Pair<RexNode, String> get(int index) {
                if (index < rightCount) {
                    RelDataTypeField field = fields.get(index);
                    return Pair.<RexNode, String>of(new RexInputRef(index, field.getType()), field.getName());
                } else {
                    return Pair.of(RexUtil.shift(extraRightExprs.get(index - rightCount), -newLeftCount), null);
                }
            }
        };
        relBuilder.project(Pair.left(pairs), Pair.right(pairs));
    }
    final RelNode right = relBuilder.build();
    final RelNode left = relBuilder.build();
    relBuilder.push(originalJoin.copy(originalJoin.getTraitSet(), joinCond, left, right, joinType, originalJoin.isSemiJoinDone()));
    if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) {
        Mappings.TargetMapping mapping = Mappings.createShiftMapping(leftCount + extraLeftExprs.size() + rightCount + extraRightExprs.size(), 0, 0, leftCount, leftCount, leftCount + extraLeftExprs.size(), rightCount);
        relBuilder.project(relBuilder.fields(mapping.inverse()));
    }
    return relBuilder.build();
}

Example 84 with Pair

use of org.apache.calcite.util.Pair in project calcite by apache.

the class RelOptUtil method createProjectJoinRel.

// to be removed before 2.0
@Deprecated
public static RelNode createProjectJoinRel(List<Integer> outputProj, RelNode joinRel) {
    int newProjectOutputSize = outputProj.size();
    List<RelDataTypeField> joinOutputFields = joinRel.getRowType().getFieldList();
    // join, then no need to create a projection
    if ((newProjectOutputSize > 0) && (newProjectOutputSize < joinOutputFields.size())) {
        final List<Pair<RexNode, String>> newProjects = new ArrayList<>();
        final RelBuilder relBuilder = RelFactories.LOGICAL_BUILDER.create(joinRel.getCluster(), null);
        final RexBuilder rexBuilder = relBuilder.getRexBuilder();
        for (int fieldIndex : outputProj) {
            final RelDataTypeField field = joinOutputFields.get(fieldIndex);
            newProjects.add(Pair.<RexNode, String>of(rexBuilder.makeInputRef(field.getType(), fieldIndex), field.getName()));
        }
        // Create a project rel on the output of the join.
        return relBuilder.push(joinRel).project(Pair.left(newProjects), Pair.right(newProjects), true).build();
    }
    return joinRel;
}

Example 85 with Pair

use of org.apache.calcite.util.Pair in project calcite by apache.

the class RexImplicationChecker method implies2.

/**
 * Returns whether the predicate {@code first} (not a conjunction)
 * implies {@code second}.
 */
private boolean implies2(RexNode first, RexNode second) {
    if (second.isAlwaysFalse()) {
        // f cannot imply s
        return false;
    }
    // E.g. "x is null" implies "x is null".
    if (RexUtil.eq(first, second)) {
        return true;
    }
    // Several things imply "IS NOT NULL"
    switch(second.getKind()) {
        case IS_NOT_NULL:
            // Suppose we know that first is strong in second; that is,
            // the if second is null, then first will be null.
            // Then, first being not null implies that second is not null.
            // 
            // For example, first is "x > y", second is "x".
            // If we know that "x > y" is not null, we know that "x" is not null.
            final RexNode operand = ((RexCall) second).getOperands().get(0);
            final Strong strong = new Strong() {

                @Override
                public boolean isNull(RexNode node) {
                    return RexUtil.eq(node, operand) || super.isNull(node);
                }
            };
            if (strong.isNull(first)) {
                return true;
            }
    }
    final InputUsageFinder firstUsageFinder = new InputUsageFinder();
    final InputUsageFinder secondUsageFinder = new InputUsageFinder();
    RexUtil.apply(firstUsageFinder, ImmutableList.<RexNode>of(), first);
    RexUtil.apply(secondUsageFinder, ImmutableList.<RexNode>of(), second);
    // Check Support
    if (!checkSupport(firstUsageFinder, secondUsageFinder)) {
        LOGGER.warn("Support for checking {} => {} is not there", first, second);
        return false;
    }
    ImmutableList.Builder<Set<Pair<RexInputRef, RexNode>>> usagesBuilder = ImmutableList.builder();
    for (Map.Entry<RexInputRef, InputRefUsage<SqlOperator, RexNode>> entry : firstUsageFinder.usageMap.entrySet()) {
        ImmutableSet.Builder<Pair<RexInputRef, RexNode>> usageBuilder = ImmutableSet.builder();
        if (entry.getValue().usageList.size() > 0) {
            for (final Pair<SqlOperator, RexNode> pair : entry.getValue().usageList) {
                usageBuilder.add(Pair.of(entry.getKey(), pair.getValue()));
            }
            usagesBuilder.add(usageBuilder.build());
        }
    }
    final Set<List<Pair<RexInputRef, RexNode>>> usages = Sets.cartesianProduct(usagesBuilder.build());
    for (List<Pair<RexInputRef, RexNode>> usageList : usages) {
        // Get the literals from first conjunction and executes second conjunction
        // using them.
        // 
        // E.g., for
        // x > 30 &rArr; x > 10,
        // we will replace x by 30 in second expression and execute it i.e.,
        // 30 > 10
        // 
        // If it's true then we infer implication.
        final DataContext dataValues = VisitorDataContext.of(rowType, usageList);
        if (!isSatisfiable(second, dataValues)) {
            return false;
        }
    }
    return true;
}