Examples with HiveParserRowResolver org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver used on opensource projects

Example 1 with HiveParserRowResolver

use of org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver in project flink by apache.

the class HiveParserCalcitePlanner method genSelectForWindowing.

private RelNode genSelectForWindowing(HiveParserQB qb, RelNode srcRel, HashSet<ColumnInfo> newColumns) throws SemanticException {
    HiveParserWindowingSpec wSpec = !qb.getAllWindowingSpecs().isEmpty() ? qb.getAllWindowingSpecs().values().iterator().next() : null;
    if (wSpec == null) {
        return null;
    }
    // 1. Get valid Window Function Spec
    wSpec.validateAndMakeEffective();
    List<HiveParserWindowingSpec.WindowExpressionSpec> windowExpressions = wSpec.getWindowExpressions();
    if (windowExpressions == null || windowExpressions.isEmpty()) {
        return null;
    }
    HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
    // 2. Get RexNodes for original Projections from below
    List<RexNode> projsForWindowSelOp = new ArrayList<>(HiveParserUtils.getProjsFromBelowAsInputRef(srcRel));
    // 3. Construct new Row Resolver with everything from below.
    HiveParserRowResolver outRR = new HiveParserRowResolver();
    if (!HiveParserRowResolver.add(outRR, inputRR)) {
        LOG.warn("Duplicates detected when adding columns to RR: see previous message");
    }
    // 4. Walk through Window Expressions & Construct RexNodes for those. Update out_rwsch
    final HiveParserQBParseInfo qbp = qb.getParseInfo();
    final String selClauseName = qbp.getClauseNames().iterator().next();
    final boolean cubeRollupGrpSetPresent = (!qbp.getDestRollups().isEmpty() || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty());
    for (HiveParserWindowingSpec.WindowExpressionSpec winExprSpec : windowExpressions) {
        if (!qbp.getDestToGroupBy().isEmpty()) {
            // Special handling of grouping function
            winExprSpec.setExpression(rewriteGroupingFunctionAST(getGroupByForClause(qbp, selClauseName), winExprSpec.getExpression(), !cubeRollupGrpSetPresent));
        }
        if (outRR.getExpression(winExprSpec.getExpression()) == null) {
            Pair<RexNode, TypeInfo> rexAndType = getWindowRexAndType(winExprSpec, srcRel);
            projsForWindowSelOp.add(rexAndType.getKey());
            // 6.2.2 Update Output Row Schema
            ColumnInfo oColInfo = new ColumnInfo(getColumnInternalName(projsForWindowSelOp.size()), rexAndType.getValue(), null, false);
            outRR.putExpression(winExprSpec.getExpression(), oColInfo);
            newColumns.add(oColInfo);
        }
    }
    return genSelectRelNode(projsForWindowSelOp, outRR, srcRel, windowExpressions);
}

Example 2 with HiveParserRowResolver

use of org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver in project flink by apache.

the class HiveParserCalcitePlanner method genLateralViewPlan.

private RelNode genLateralViewPlan(HiveParserQB qb, Map<String, RelNode> aliasToRel) throws SemanticException {
    Map<String, ArrayList<HiveParserASTNode>> aliasToLateralViews = qb.getParseInfo().getAliasToLateralViews();
    Preconditions.checkArgument(aliasToLateralViews.size() == 1, "We only support lateral views for 1 alias");
    Map.Entry<String, ArrayList<HiveParserASTNode>> entry = aliasToLateralViews.entrySet().iterator().next();
    String alias = entry.getKey();
    RelNode res = null;
    List<HiveParserASTNode> lateralViews = entry.getValue();
    for (HiveParserASTNode lateralView : lateralViews) {
        Preconditions.checkArgument(lateralView.getChildCount() == 2);
        final boolean isOuter = lateralView.getType() == HiveASTParser.TOK_LATERAL_VIEW_OUTER;
        // this is the 1st lateral view
        if (res == null) {
            // LHS can be table or sub-query
            res = aliasToRel.get(alias);
        }
        Preconditions.checkState(res != null, "Failed to decide LHS table for current lateral view");
        HiveParserRowResolver inputRR = relToRowResolver.get(res);
        HiveParserUtils.LateralViewInfo info = HiveParserUtils.extractLateralViewInfo(lateralView, inputRR, semanticAnalyzer, frameworkConfig, cluster);
        HiveParserRexNodeConverter rexNodeConverter = new HiveParserRexNodeConverter(cluster, res.getRowType(), relToHiveColNameCalcitePosMap.get(res), 0, false, funcConverter);
        List<RexNode> operands = new ArrayList<>(info.getOperands().size());
        for (ExprNodeDesc exprDesc : info.getOperands()) {
            operands.add(rexNodeConverter.convert(exprDesc).accept(funcConverter));
        }
        res = genUDTFPlan(info.getSqlOperator(), info.getFuncName(), info.getTabAlias(), info.getColAliases(), qb, operands, info.getOperandColInfos(), res, false, isOuter);
    }
    return res;
}

Example 3 with HiveParserRowResolver

use of org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver in project flink by apache.

the class HiveParserCalcitePlanner method genLimitLogicalPlan.

private Sort genLimitLogicalPlan(HiveParserQB qb, RelNode srcRel) throws SemanticException {
    Sort sortRel = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    AbstractMap.SimpleEntry<Integer, Integer> entry = qbp.getDestToLimit().get(qbp.getClauseNames().iterator().next());
    int offset = (entry == null) ? 0 : entry.getKey();
    Integer fetch = (entry == null) ? null : entry.getValue();
    if (fetch != null) {
        RexNode offsetRex = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(offset));
        RexNode fetchRex = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(fetch));
        RelTraitSet traitSet = cluster.traitSet();
        RelCollation canonizedCollation = traitSet.canonize(RelCollations.EMPTY);
        sortRel = LogicalSort.create(srcRel, canonizedCollation, offsetRex, fetchRex);
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        if (!HiveParserRowResolver.add(outputRR, relToRowResolver.get(srcRel))) {
            throw new SemanticException("Duplicates detected when adding columns to RR: see previous message");
        }
        Map<String, Integer> hiveColNameCalcitePosMap = buildHiveToCalciteColumnMap(outputRR);
        relToRowResolver.put(sortRel, outputRR);
        relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
    }
    return sortRel;
}

Example 4 with HiveParserRowResolver

use of org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver in project flink by apache.

the class HiveParserCalcitePlanner method genGBLogicalPlan.

// Generate GB plan.
private RelNode genGBLogicalPlan(HiveParserQB qb, RelNode srcRel) throws SemanticException {
    RelNode gbRel = null;
    HiveParserQBParseInfo qbp = qb.getParseInfo();
    // 1. Gather GB Expressions (AST) (GB + Aggregations)
    // NOTE: Multi Insert is not supported
    String detsClauseName = qbp.getClauseNames().iterator().next();
    HiveParserASTNode selExprList = qb.getParseInfo().getSelForClause(detsClauseName);
    HiveParserSubQueryUtils.checkForTopLevelSubqueries(selExprList);
    if (selExprList.getToken().getType() == HiveASTParser.TOK_SELECTDI && selExprList.getChildCount() == 1 && selExprList.getChild(0).getChildCount() == 1) {
        HiveParserASTNode node = (HiveParserASTNode) selExprList.getChild(0).getChild(0);
        if (node.getToken().getType() == HiveASTParser.TOK_ALLCOLREF) {
            srcRel = genSelectLogicalPlan(qb, srcRel, srcRel, null, null);
            HiveParserRowResolver rr = relToRowResolver.get(srcRel);
            qbp.setSelExprForClause(detsClauseName, HiveParserUtils.genSelectDIAST(rr));
        }
    }
    // Select DISTINCT + windowing; GBy handled by genSelectForWindowing
    if (selExprList.getToken().getType() == HiveASTParser.TOK_SELECTDI && !qb.getAllWindowingSpecs().isEmpty()) {
        return null;
    }
    List<HiveParserASTNode> gbAstExprs = getGroupByForClause(qbp, detsClauseName);
    HashMap<String, HiveParserASTNode> aggregationTrees = qbp.getAggregationExprsForClause(detsClauseName);
    boolean hasGrpByAstExprs = !gbAstExprs.isEmpty();
    boolean hasAggregationTrees = aggregationTrees != null && !aggregationTrees.isEmpty();
    final boolean cubeRollupGrpSetPresent = !qbp.getDestRollups().isEmpty() || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty();
    // 2. Sanity check
    if (semanticAnalyzer.getConf().getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW) && qbp.getDistinctFuncExprsForClause(detsClauseName).size() > 1) {
        throw new SemanticException(ErrorMsg.UNSUPPORTED_MULTIPLE_DISTINCTS.getMsg());
    }
    if (hasGrpByAstExprs || hasAggregationTrees) {
        ArrayList<ExprNodeDesc> gbExprNodeDescs = new ArrayList<>();
        ArrayList<String> outputColNames = new ArrayList<>();
        // 3. Input, Output Row Resolvers
        HiveParserRowResolver inputRR = relToRowResolver.get(srcRel);
        HiveParserRowResolver outputRR = new HiveParserRowResolver();
        outputRR.setIsExprResolver(true);
        if (hasGrpByAstExprs) {
            // 4. Construct GB Keys (ExprNode)
            for (HiveParserASTNode gbAstExpr : gbAstExprs) {
                Map<HiveParserASTNode, ExprNodeDesc> astToExprNodeDesc = semanticAnalyzer.genAllExprNodeDesc(gbAstExpr, inputRR);
                ExprNodeDesc grpbyExprNDesc = astToExprNodeDesc.get(gbAstExpr);
                if (grpbyExprNDesc == null) {
                    throw new SemanticException("Invalid Column Reference: " + gbAstExpr.dump());
                }
                addToGBExpr(outputRR, inputRR, gbAstExpr, grpbyExprNDesc, gbExprNodeDescs, outputColNames);
            }
        }
        // 5. GroupingSets, Cube, Rollup
        int numGroupCols = gbExprNodeDescs.size();
        List<Integer> groupingSets = null;
        if (cubeRollupGrpSetPresent) {
            if (qbp.getDestRollups().contains(detsClauseName)) {
                groupingSets = getGroupingSetsForRollup(gbAstExprs.size());
            } else if (qbp.getDestCubes().contains(detsClauseName)) {
                groupingSets = getGroupingSetsForCube(gbAstExprs.size());
            } else if (qbp.getDestGroupingSets().contains(detsClauseName)) {
                groupingSets = getGroupingSets(gbAstExprs, qbp, detsClauseName);
            }
        }
        // 6. Construct aggregation function Info
        ArrayList<AggInfo> aggInfos = new ArrayList<>();
        if (hasAggregationTrees) {
            for (HiveParserASTNode value : aggregationTrees.values()) {
                // 6.1 Determine type of UDAF
                // This is the GenericUDAF name
                String aggName = unescapeIdentifier(value.getChild(0).getText());
                boolean isDistinct = value.getType() == HiveASTParser.TOK_FUNCTIONDI;
                boolean isAllColumns = value.getType() == HiveASTParser.TOK_FUNCTIONSTAR;
                // 6.2 Convert UDAF Params to ExprNodeDesc
                ArrayList<ExprNodeDesc> aggParameters = new ArrayList<>();
                for (int i = 1; i < value.getChildCount(); i++) {
                    HiveParserASTNode paraExpr = (HiveParserASTNode) value.getChild(i);
                    ExprNodeDesc paraExprNode = semanticAnalyzer.genExprNodeDesc(paraExpr, inputRR);
                    aggParameters.add(paraExprNode);
                }
                GenericUDAFEvaluator.Mode aggMode = HiveParserUtils.groupByDescModeToUDAFMode(GroupByDesc.Mode.COMPLETE, isDistinct);
                GenericUDAFEvaluator genericUDAFEvaluator = HiveParserUtils.getGenericUDAFEvaluator(aggName, aggParameters, value, isDistinct, isAllColumns, frameworkConfig.getOperatorTable());
                assert (genericUDAFEvaluator != null);
                HiveParserBaseSemanticAnalyzer.GenericUDAFInfo udaf = HiveParserUtils.getGenericUDAFInfo(genericUDAFEvaluator, aggMode, aggParameters);
                String aggAlias = null;
                if (value.getParent().getType() == HiveASTParser.TOK_SELEXPR && value.getParent().getChildCount() == 2) {
                    aggAlias = unescapeIdentifier(value.getParent().getChild(1).getText().toLowerCase());
                }
                AggInfo aggInfo = new AggInfo(aggParameters, udaf.returnType, aggName, isDistinct, isAllColumns, aggAlias);
                aggInfos.add(aggInfo);
                String field = aggAlias == null ? getColumnInternalName(numGroupCols + aggInfos.size() - 1) : aggAlias;
                outputColNames.add(field);
                outputRR.putExpression(value, new ColumnInfo(field, aggInfo.getReturnType(), "", false));
            }
        }
        // grouping sets
        if (groupingSets != null && !groupingSets.isEmpty()) {
            String field = getColumnInternalName(numGroupCols + aggInfos.size());
            outputColNames.add(field);
            outputRR.put(null, VirtualColumn.GROUPINGID.getName(), new ColumnInfo(field, // flink grouping_id's return type is bigint
            TypeInfoFactory.longTypeInfo, null, true));
        }
        // 8. We create the group_by operator
        gbRel = genGBRelNode(gbExprNodeDescs, aggInfos, groupingSets, srcRel);
        relToHiveColNameCalcitePosMap.put(gbRel, buildHiveToCalciteColumnMap(outputRR));
        relToRowResolver.put(gbRel, outputRR);
    }
    return gbRel;
}

Example 5 with HiveParserRowResolver

use of org.apache.flink.table.planner.delegation.hive.copy.HiveParserRowResolver in project flink by apache.

the class HiveParserCalcitePlanner method genSetOpLogicalPlan.

@SuppressWarnings("nls")
private RelNode genSetOpLogicalPlan(HiveParserQBExpr.Opcode opcode, String alias, String leftalias, RelNode leftRel, String rightalias, RelNode rightRel) throws SemanticException {
    // 1. Get Row Resolvers, Column map for original left and right input of SetOp Rel
    HiveParserRowResolver leftRR = relToRowResolver.get(leftRel);
    HiveParserRowResolver rightRR = relToRowResolver.get(rightRel);
    HashMap<String, ColumnInfo> leftMap = leftRR.getFieldMap(leftalias);
    HashMap<String, ColumnInfo> rightMap = rightRR.getFieldMap(rightalias);
    // 2. Validate that SetOp is feasible according to Hive (by using type info from RR)
    if (leftMap.size() != rightMap.size()) {
        throw new SemanticException("Schema of both sides of union should match.");
    }
    // 3. construct SetOp Output RR using original left & right Input
    HiveParserRowResolver setOpOutRR = new HiveParserRowResolver();
    Iterator<Map.Entry<String, ColumnInfo>> lIter = leftMap.entrySet().iterator();
    Iterator<Map.Entry<String, ColumnInfo>> rIter = rightMap.entrySet().iterator();
    while (lIter.hasNext()) {
        Map.Entry<String, ColumnInfo> lEntry = lIter.next();
        Map.Entry<String, ColumnInfo> rEntry = rIter.next();
        ColumnInfo lInfo = lEntry.getValue();
        ColumnInfo rInfo = rEntry.getValue();
        String field = lEntry.getKey();
        // try widening conversion, otherwise fail union
        TypeInfo commonTypeInfo = FunctionRegistry.getCommonClassForUnionAll(lInfo.getType(), rInfo.getType());
        if (commonTypeInfo == null) {
            HiveParserASTNode tabRef = getQB().getAliases().isEmpty() ? null : getQB().getParseInfo().getSrcForAlias(getQB().getAliases().get(0));
            throw new SemanticException(generateErrorMessage(tabRef, "Schema of both sides of setop should match: Column " + field + " is of type " + lInfo.getType().getTypeName() + " on first table and type " + rInfo.getType().getTypeName() + " on second table"));
        }
        ColumnInfo setOpColInfo = new ColumnInfo(lInfo);
        setOpColInfo.setType(commonTypeInfo);
        setOpOutRR.put(alias, field, setOpColInfo);
    }
    // 4. Determine which columns requires cast on left/right input (Calcite requires exact
    // types on both sides of SetOp)
    boolean leftNeedsTypeCast = false;
    boolean rightNeedsTypeCast = false;
    List<RexNode> leftProjs = new ArrayList<>();
    List<RexNode> rightProjs = new ArrayList<>();
    List<RelDataTypeField> leftFields = leftRel.getRowType().getFieldList();
    List<RelDataTypeField> rightFields = rightRel.getRowType().getFieldList();
    for (int i = 0; i < leftFields.size(); i++) {
        RelDataType leftFieldType = leftFields.get(i).getType();
        RelDataType rightFieldType = rightFields.get(i).getType();
        if (!leftFieldType.equals(rightFieldType)) {
            RelDataType unionFieldType = HiveParserUtils.toRelDataType(setOpOutRR.getColumnInfos().get(i).getType(), cluster.getTypeFactory());
            if (!unionFieldType.equals(leftFieldType)) {
                leftNeedsTypeCast = true;
            }
            leftProjs.add(cluster.getRexBuilder().ensureType(unionFieldType, cluster.getRexBuilder().makeInputRef(leftFieldType, i), true));
            if (!unionFieldType.equals(rightFieldType)) {
                rightNeedsTypeCast = true;
            }
            rightProjs.add(cluster.getRexBuilder().ensureType(unionFieldType, cluster.getRexBuilder().makeInputRef(rightFieldType, i), true));
        } else {
            leftProjs.add(cluster.getRexBuilder().ensureType(leftFieldType, cluster.getRexBuilder().makeInputRef(leftFieldType, i), true));
            rightProjs.add(cluster.getRexBuilder().ensureType(rightFieldType, cluster.getRexBuilder().makeInputRef(rightFieldType, i), true));
        }
    }
    // parity
    if (leftNeedsTypeCast) {
        leftRel = LogicalProject.create(leftRel, Collections.emptyList(), leftProjs, leftRel.getRowType().getFieldNames());
    }
    if (rightNeedsTypeCast) {
        rightRel = LogicalProject.create(rightRel, Collections.emptyList(), rightProjs, rightRel.getRowType().getFieldNames());
    }
    // 6. Construct SetOp Rel
    List<RelNode> leftAndRight = Arrays.asList(leftRel, rightRel);
    SetOp setOpRel;
    switch(opcode) {
        case UNION:
            setOpRel = LogicalUnion.create(leftAndRight, true);
            break;
        case INTERSECT:
            setOpRel = LogicalIntersect.create(leftAndRight, false);
            break;
        case INTERSECTALL:
            setOpRel = LogicalIntersect.create(leftAndRight, true);
            break;
        case EXCEPT:
            setOpRel = LogicalMinus.create(leftAndRight, false);
            break;
        case EXCEPTALL:
            setOpRel = LogicalMinus.create(leftAndRight, true);
            break;
        default:
            throw new SemanticException("Unsupported set operator " + opcode.toString());
    }
    relToRowResolver.put(setOpRel, setOpOutRR);
    relToHiveColNameCalcitePosMap.put(setOpRel, buildHiveToCalciteColumnMap(setOpOutRR));
    return setOpRel;
}