Examples with Node org.apache.hadoop.hive.ql.lib.Node used on opensource projects

Example 1 with Node

use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.

the class AnnotateReduceSinkOutputOperator method transform.

@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
    // 1. We apply the transformation
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", "(" + ReduceSinkOperator.getOperatorName() + "%)"), new ReduceSinkOutputOperatorAnnotator());
    GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null));
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getTopOps().values());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 2 with Node

use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.

the class CrossProductCheck method resolve.

@Override
public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
    TaskGraphWalker ogw = new TaskGraphWalker(this);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pctx.getRootTasks());
    ogw.startWalking(topNodes, null);
    return pctx;
}

Example 3 with Node

use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.

the class SemanticAnalyzer method walkASTMarkTABREF.

private void walkASTMarkTABREF(ASTNode ast, Set<String> cteAlias) throws SemanticException {
    Queue<Node> queue = new LinkedList<>();
    queue.add(ast);
    Map<HivePrivilegeObject, MaskAndFilterInfo> basicInfos = new LinkedHashMap<>();
    while (!queue.isEmpty()) {
        ASTNode astNode = (ASTNode) queue.poll();
        if (astNode.getToken().getType() == HiveParser.TOK_TABREF) {
            int aliasIndex = 0;
            StringBuffer additionalTabInfo = new StringBuffer();
            for (int index = 1; index < astNode.getChildCount(); index++) {
                ASTNode ct = (ASTNode) astNode.getChild(index);
                if (ct.getToken().getType() == HiveParser.TOK_TABLEBUCKETSAMPLE || ct.getToken().getType() == HiveParser.TOK_TABLESPLITSAMPLE || ct.getToken().getType() == HiveParser.TOK_TABLEPROPERTIES) {
                    additionalTabInfo.append(ctx.getTokenRewriteStream().toString(ct.getTokenStartIndex(), ct.getTokenStopIndex()));
                } else {
                    aliasIndex = index;
                }
            }
            ASTNode tableTree = (ASTNode) (astNode.getChild(0));
            String tabIdName = getUnescapedName(tableTree);
            String alias;
            if (aliasIndex != 0) {
                alias = unescapeIdentifier(astNode.getChild(aliasIndex).getText());
            } else {
                alias = getUnescapedUnqualifiedTableName(tableTree);
            }
            // select * from TAB2 [no masking]
            if (cteAlias.contains(tabIdName)) {
                continue;
            }
            String replacementText = null;
            Table table = null;
            try {
                table = getTableObjectByName(tabIdName);
            } catch (HiveException e) {
                // Table may not be found when materialization of CTE is on.
                LOG.info("Table " + tabIdName + " is not found in walkASTMarkTABREF.");
                continue;
            }
            List<String> colNames = new ArrayList<>();
            List<String> colTypes = new ArrayList<>();
            for (FieldSchema col : table.getAllCols()) {
                colNames.add(col.getName());
                colTypes.add(col.getType());
            }
            basicInfos.put(new HivePrivilegeObject(table.getDbName(), table.getTableName(), colNames), new MaskAndFilterInfo(colTypes, additionalTabInfo.toString(), alias, astNode, table.isView()));
        }
        if (astNode.getChildCount() > 0 && !ignoredTokens.contains(astNode.getToken().getType())) {
            for (Node child : astNode.getChildren()) {
                queue.offer(child);
            }
        }
    }
    List<HivePrivilegeObject> basicPrivObjs = new ArrayList<>();
    basicPrivObjs.addAll(basicInfos.keySet());
    List<HivePrivilegeObject> needRewritePrivObjs = tableMask.applyRowFilterAndColumnMasking(basicPrivObjs);
    if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) {
        for (HivePrivilegeObject privObj : needRewritePrivObjs) {
            MaskAndFilterInfo info = basicInfos.get(privObj);
            String replacementText = tableMask.create(privObj, info);
            if (replacementText != null) {
                // We don't support masking/filtering against ACID query at the moment
                if (ctx.getIsUpdateDeleteMerge()) {
                    throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED, privObj.getDbname(), privObj.getObjectName());
                }
                tableMask.setNeedsRewrite(true);
                tableMask.addTranslation(info.astNode, replacementText);
            }
        }
    }
}

Example 4 with Node

use of org.apache.hadoop.hive.ql.lib.Node in project hive by apache.

the class MapReduceCompiler method generateTaskTree.

@Override
protected void generateTaskTree(List<Task<? extends Serializable>> rootTasks, ParseContext pCtx, List<Task<MoveWork>> mvTask, Set<ReadEntity> inputs, Set<WriteEntity> outputs) throws SemanticException {
    // generate map reduce plans
    ParseContext tempParseContext = getParseContext(pCtx, rootTasks);
    GenMRProcContext procCtx = new GenMRProcContext(conf, // Must be deterministic order map for consistent q-test output across Java versions
    new LinkedHashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>>(), tempParseContext, mvTask, rootTasks, new LinkedHashMap<Operator<? extends OperatorDesc>, GenMapRedCtx>(), inputs, outputs);
    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack.
    // The dispatcher generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp(new String("R1"), TableScanOperator.getOperatorName() + "%"), new GenMRTableScan1());
    opRules.put(new RuleRegExp(new String("R2"), TableScanOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink1());
    opRules.put(new RuleRegExp(new String("R3"), ReduceSinkOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink2());
    opRules.put(new RuleRegExp(new String("R4"), FileSinkOperator.getOperatorName() + "%"), new GenMRFileSink1());
    opRules.put(new RuleRegExp(new String("R5"), UnionOperator.getOperatorName() + "%"), new GenMRUnion1());
    opRules.put(new RuleRegExp(new String("R6"), UnionOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"), new GenMRRedSink3());
    opRules.put(new RuleRegExp(new String("R7"), MapJoinOperator.getOperatorName() + "%"), MapJoinFactory.getTableScanMapJoin());
    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules, procCtx);
    GraphWalker ogw = new GenMapRedWalker(disp);
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(pCtx.getTopOps().values());
    ogw.startWalking(topNodes, null);
}

Example 5 with Node

use of org.apache.hadoop.hive.ql.lib.Node in project SQLWindowing by hbutani.

the class WindowingTypeCheckProcFactory method genExprNode.

public static HashMap<Node, Object> genExprNode(ASTNode expr, TypeCheckCtx tcCtx) throws SemanticException {
    // Create the walker, the rules dispatcher and the context.
    // create a walker which walks the tree in a DFS manner while
    // maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", Windowing2Parser.NULL + "%"), getNullExprProcessor());
    opRules.put(new RuleRegExp("R2", Windowing2Parser.Number + "%|" + Windowing2Parser.TinyintLiteral + "%|" + Windowing2Parser.SmallintLiteral + "%|" + Windowing2Parser.BigintLiteral + "%"), getNumExprProcessor());
    opRules.put(new RuleRegExp("R3", Windowing2Parser.Identifier + "%|" + Windowing2Parser.StringLiteral + "%|" + Windowing2Parser.CHARSETLITERAL + "%|" + Windowing2Parser.STRINGLITERALSEQUENCE + "%|" + "%|" + Windowing2Parser.IF + "%|" + Windowing2Parser.CASE + "%|" + Windowing2Parser.WHEN + "%|" + Windowing2Parser.IN + "%|" + Windowing2Parser.ARRAY + "%|" + Windowing2Parser.MAP + "%|" + Windowing2Parser.BETWEEN + "%|" + Windowing2Parser.STRUCT + "%"), getStrExprProcessor());
    opRules.put(new RuleRegExp("R4", Windowing2Parser.TRUE + "%|" + Windowing2Parser.FALSE + "%"), getBoolExprProcessor());
    opRules.put(new RuleRegExp("R5", Windowing2Parser.TABLEORCOL + "%"), getColumnExprProcessor());
    // The dispatcher fires the processor corresponding to the closest
    // matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), opRules, tcCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.add(expr);
    HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
    ogw.startWalking(topNodes, nodeOutputs);
    return nodeOutputs;
}