Examples with TreeVisitorAction org.antlr.runtime.tree.TreeVisitorAction used on opensource projects

Example 1 with TreeVisitorAction

use of org.antlr.runtime.tree.TreeVisitorAction in project antlr4 by tunnelvisionlabs.

the class GrammarTransformPipeline method expandParameterizedLoops.

/**
 * Find and replace
 *      ID*[','] with ID (',' ID)*
 *      ID+[','] with ID (',' ID)+
 *      (x {action} y)+[','] with x {action} y (',' x {action} y)+
 *
 *  Parameter must be a token.
 *  todo: do we want?
 */
public void expandParameterizedLoops(GrammarAST root) {
    TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
    v.visit(root, new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            if (((GrammarAST) t).getType() == 3) {
                return expandParameterizedLoop((GrammarAST) t);
            }
            return t;
        }

        @Override
        public Object post(Object t) {
            return t;
        }
    });
}

Example 2 with TreeVisitorAction

use of org.antlr.runtime.tree.TreeVisitorAction in project flink by apache.

the class HiveParserUtils method rewriteGroupingFunctionAST.

public static HiveParserASTNode rewriteGroupingFunctionAST(final List<HiveParserASTNode> grpByAstExprs, HiveParserASTNode targetNode, final boolean noneSet) throws SemanticException {
    final MutableBoolean visited = new MutableBoolean(false);
    final MutableBoolean found = new MutableBoolean(false);
    final boolean legacyGrouping = legacyGrouping();
    TreeVisitorAction action = new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            return t;
        }

        @Override
        public Object post(Object t) {
            HiveParserASTNode current = (HiveParserASTNode) t;
            // rewrite grouping function
            if (current.getType() == HiveASTParser.TOK_FUNCTION && current.getChildCount() >= 2) {
                HiveParserASTNode func = (HiveParserASTNode) current.getChild(0);
                if (func.getText().equals("grouping")) {
                    visited.setValue(true);
                    convertGrouping(current, grpByAstExprs, noneSet, legacyGrouping, found);
                }
            } else if (legacyGrouping && current.getType() == HiveASTParser.TOK_TABLE_OR_COL && current.getChildCount() == 1) {
                // rewrite grouping__id
                HiveParserASTNode child = (HiveParserASTNode) current.getChild(0);
                if (child.getText().equalsIgnoreCase(VirtualColumn.GROUPINGID.getName())) {
                    return convertToLegacyGroupingId(current, grpByAstExprs.size());
                }
            }
            return t;
        }
    };
    HiveParserASTNode newTargetNode = (HiveParserASTNode) new TreeVisitor(HiveASTParseDriver.ADAPTOR).visit(targetNode, action);
    if (visited.booleanValue() && !found.booleanValue()) {
        throw new SemanticException("Expression in GROUPING function not present in GROUP BY");
    }
    return newTargetNode;
}

Example 3 with TreeVisitorAction

use of org.antlr.runtime.tree.TreeVisitorAction in project antlr4 by antlr.

the class GrammarTransformPipeline method setGrammarPtr.

/**
 * Utility visitor that sets grammar ptr in each node
 */
public static void setGrammarPtr(final Grammar g, GrammarAST tree) {
    if (tree == null)
        return;
    // ensure each node has pointer to surrounding grammar
    TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
    v.visit(tree, new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            ((GrammarAST) t).g = g;
            return t;
        }

        @Override
        public Object post(Object t) {
            return t;
        }
    });
}

Example 4 with TreeVisitorAction

use of org.antlr.runtime.tree.TreeVisitorAction in project hive by apache.

the class CalcitePlanner method rewriteASTForMultiInsert.

private ASTNode rewriteASTForMultiInsert(ASTNode query, ASTNode nodeOfInterest) {
    // 1. gather references from original query
    // This is a map from aliases to references.
    // We keep all references as we will need to modify them after creating
    // the subquery
    final Multimap<String, Object> aliasNodes = ArrayListMultimap.create();
    // To know if we need to bail out
    final AtomicBoolean notSupported = new AtomicBoolean(false);
    TreeVisitorAction action = new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            if (!notSupported.get()) {
                if (ParseDriver.adaptor.getType(t) == HiveParser.TOK_ALLCOLREF) {
                    // TODO: this is a limitation of the AST rewriting approach that we will
                    // not be able to overcome till proper integration of full multi-insert
                    // queries with Calcite is implemented.
                    // The current rewriting gather references from insert clauses and then
                    // updates them with the new subquery references. However, if insert
                    // clauses use * or tab.*, we cannot resolve the columns that we are
                    // referring to. Thus, we just bail out and those queries will not be
                    // currently optimized by Calcite.
                    // An example of such query is:
                    // FROM T_A a LEFT JOIN T_B b ON a.id = b.id
                    // INSERT OVERWRITE TABLE join_result_1
                    // SELECT a.*, b.*
                    // INSERT OVERWRITE TABLE join_result_3
                    // SELECT a.*, b.*;
                    notSupported.set(true);
                } else if (ParseDriver.adaptor.getType(t) == HiveParser.DOT) {
                    Object c = ParseDriver.adaptor.getChild(t, 0);
                    if (c != null && ParseDriver.adaptor.getType(c) == HiveParser.TOK_TABLE_OR_COL) {
                        aliasNodes.put(((ASTNode) t).toStringTree(), t);
                    }
                } else if (ParseDriver.adaptor.getType(t) == HiveParser.TOK_TABLE_OR_COL) {
                    Object p = ParseDriver.adaptor.getParent(t);
                    if (p == null || ParseDriver.adaptor.getType(p) != HiveParser.DOT) {
                        aliasNodes.put(((ASTNode) t).toStringTree(), t);
                    }
                }
            }
            return t;
        }

        @Override
        public Object post(Object t) {
            return t;
        }
    };
    TreeVisitor tv = new TreeVisitor(ParseDriver.adaptor);
    // the subtree to gather the references
    for (int i = 0; i < query.getChildCount(); i++) {
        ASTNode child = (ASTNode) query.getChild(i);
        if (ParseDriver.adaptor.getType(child) != HiveParser.TOK_INSERT) {
            // If it is not an INSERT, we do not need to anything
            continue;
        }
        tv.visit(child, action);
    }
    if (notSupported.get()) {
        // Bail out
        return null;
    }
    // 2. rewrite into query
    // TOK_QUERY
    // TOK_FROM
    // join
    // TOK_INSERT
    // TOK_DESTINATION
    // TOK_DIR
    // TOK_TMP_FILE
    // TOK_SELECT
    // refs
    ASTNode from = new ASTNode(FROM_TOKEN);
    from.addChild((ASTNode) ParseDriver.adaptor.dupTree(nodeOfInterest));
    ASTNode destination = new ASTNode(DEST_TOKEN);
    ASTNode dir = new ASTNode(DIR_TOKEN);
    ASTNode tmpFile = new ASTNode(TMPFILE_TOKEN);
    dir.addChild(tmpFile);
    destination.addChild(dir);
    ASTNode select = new ASTNode(SELECT_TOKEN);
    int num = 0;
    for (Collection<Object> selectIdentifier : aliasNodes.asMap().values()) {
        Iterator<Object> it = selectIdentifier.iterator();
        ASTNode node = (ASTNode) it.next();
        // Add select expression
        ASTNode selectExpr = new ASTNode(SELEXPR_TOKEN);
        // Identifier
        selectExpr.addChild((ASTNode) ParseDriver.adaptor.dupTree(node));
        String colAlias = "col" + num;
        // Alias
        selectExpr.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, colAlias)));
        select.addChild(selectExpr);
        // Rewrite all INSERT references (all the node values for this key)
        ASTNode colExpr = new ASTNode(TABLEORCOL_TOKEN);
        colExpr.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, colAlias)));
        replaceASTChild(node, colExpr);
        while (it.hasNext()) {
            // Loop to rewrite rest of INSERT references
            node = (ASTNode) it.next();
            colExpr = new ASTNode(TABLEORCOL_TOKEN);
            colExpr.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, colAlias)));
            replaceASTChild(node, colExpr);
        }
        num++;
    }
    ASTNode insert = new ASTNode(INSERT_TOKEN);
    insert.addChild(destination);
    insert.addChild(select);
    ASTNode newQuery = new ASTNode(QUERY_TOKEN);
    newQuery.addChild(from);
    newQuery.addChild(insert);
    // 3. create subquery
    ASTNode subq = new ASTNode(SUBQUERY_TOKEN);
    subq.addChild(newQuery);
    subq.addChild(new ASTNode(new CommonToken(HiveParser.Identifier, "subq")));
    replaceASTChild(nodeOfInterest, subq);
    // 4. return subquery
    return subq;
}

Example 5 with TreeVisitorAction

use of org.antlr.runtime.tree.TreeVisitorAction in project hive by apache.

the class SemanticAnalyzer method rewriteGroupingFunctionAST.

protected ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode, final boolean noneSet) {
    TreeVisitorAction action = new TreeVisitorAction() {

        @Override
        public Object pre(Object t) {
            return t;
        }

        @Override
        public Object post(Object t) {
            ASTNode root = (ASTNode) t;
            if (root.getType() == HiveParser.TOK_FUNCTION) {
                ASTNode func = (ASTNode) ParseDriver.adaptor.getChild(root, 0);
                if (func.getText().equals("grouping") && func.getChildCount() == 0) {
                    int numberOperands = ParseDriver.adaptor.getChildCount(root);
                    // We implement this logic using replaceChildren instead of replacing
                    // the root node itself because windowing logic stores multiple
                    // pointers to the AST, and replacing root might lead to some pointers
                    // leading to non-rewritten version
                    ASTNode newRoot = new ASTNode();
                    // Rewritten grouping function
                    ASTNode groupingFunc = (ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, "grouping");
                    ParseDriver.adaptor.addChild(groupingFunc, ParseDriver.adaptor.create(HiveParser.Identifier, "rewritten"));
                    newRoot.addChild(groupingFunc);
                    // Grouping ID reference
                    ASTNode childGroupingID;
                    if (noneSet) {
                        // Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus,
                        // grouping should return 0
                        childGroupingID = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, "0L");
                    } else {
                        // We refer to grouping_id column
                        childGroupingID = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
                        ParseDriver.adaptor.addChild(childGroupingID, ParseDriver.adaptor.create(HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
                    }
                    newRoot.addChild(childGroupingID);
                    // Indices
                    for (int i = 1; i < numberOperands; i++) {
                        ASTNode c = (ASTNode) ParseDriver.adaptor.getChild(root, i);
                        for (int j = 0; j < grpByAstExprs.size(); j++) {
                            ASTNode grpByExpr = grpByAstExprs.get(j);
                            if (grpByExpr.toStringTree().equals(c.toStringTree())) {
                                // Create and add AST node with position of grouping function input
                                // in group by clause
                                ASTNode childN = (ASTNode) ParseDriver.adaptor.create(HiveParser.IntegralLiteral, String.valueOf(IntMath.mod(-j - 1, grpByAstExprs.size())) + "L");
                                newRoot.addChild(childN);
                                break;
                            }
                        }
                    }
                    if (numberOperands + 1 != ParseDriver.adaptor.getChildCount(newRoot)) {
                        throw new RuntimeException(ErrorMsg.HIVE_GROUPING_FUNCTION_EXPR_NOT_IN_GROUPBY.getMsg());
                    }
                    // Replace expression
                    root.replaceChildren(0, numberOperands - 1, newRoot);
                }
            }
            return t;
        }
    };
    return (ASTNode) new TreeVisitor(ParseDriver.adaptor).visit(targetNode, action);
}