Examples with GrammarASTAdaptor org.antlr.v4.parse.GrammarASTAdaptor used on opensource projects

Example 21 with GrammarASTAdaptor

use of org.antlr.v4.parse.GrammarASTAdaptor in project antlr4 by tunnelvisionlabs.

the class LeftRecursiveRuleTransformer method translateLeftRecursiveRules.

public void translateLeftRecursiveRules() {
    String language = g.getOptionString("language");
    // translate all recursive rules
    List<String> leftRecursiveRuleNames = new ArrayList<String>();
    for (Rule r : rules) {
        if (!Grammar.isTokenName(r.name)) {
            if (LeftRecursiveRuleAnalyzer.hasImmediateRecursiveRuleRefs(r.ast, r.name)) {
                boolean fitsPattern = translateLeftRecursiveRule(ast, (LeftRecursiveRule) r, language);
                if (fitsPattern) {
                    leftRecursiveRuleNames.add(r.name);
                } else {
                // Suppressed since this build has secondary support for left recursive rules that don't
                // match the patterns for precedence rules.
                // better given an error that non-conforming left-recursion exists
                // tool.errMgr.grammarError(ErrorType.NONCONFORMING_LR_RULE, g.fileName, ((GrammarAST)r.ast.getChild(0)).token, r.name);
                }
            }
        }
    }
    // update all refs to recursive rules to have [0] argument
    for (GrammarAST r : ast.getNodesWithType(ANTLRParser.RULE_REF)) {
        // must be rule def
        if (r.getParent().getType() == ANTLRParser.RULE)
            continue;
        // already has arg; must be in rewritten rule
        if (((GrammarASTWithOptions) r).getOptionString(PRECEDENCE_OPTION_NAME) != null)
            continue;
        if (leftRecursiveRuleNames.contains(r.getText())) {
            // found ref to recursive rule not already rewritten with arg
            ((GrammarASTWithOptions) r).setOption(PRECEDENCE_OPTION_NAME, new GrammarASTAdaptor().create(ANTLRParser.INT, "0"));
        }
    }
}

Example 22 with GrammarASTAdaptor

use of org.antlr.v4.parse.GrammarASTAdaptor in project antlr4 by tunnelvisionlabs.

the class RuleFunction method getElementFrequenciesForAlt.

/**
 * Given list of X and r refs in alt, compute how many of each there are
 */
protected Tuple2<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) {
    try {
        ElementFrequenciesVisitor visitor = new ElementFrequenciesVisitor(rule.g, new CommonTreeNodeStream(new GrammarASTAdaptor(), ast));
        visitor.outerAlternative();
        if (visitor.frequencies.size() != 1) {
            factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR);
            return Tuple.create(new FrequencySet<String>(), new FrequencySet<String>());
        }
        return Tuple.create(visitor.getMinFrequencies(), visitor.frequencies.peek());
    } catch (RecognitionException ex) {
        factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, ex);
        return Tuple.create(new FrequencySet<String>(), new FrequencySet<String>());
    }
}

Example 23 with GrammarASTAdaptor

use of org.antlr.v4.parse.GrammarASTAdaptor in project antlr4 by antlr.

the class GrammarTransformPipeline method integrateImportedGrammars.

/**
 * Merge all the rules, token definitions, and named actions from
 *		imported grammars into the root grammar tree.  Perform:
 *
 *	 	(tokens { X (= Y 'y')) + (tokens { Z )	->	(tokens { X (= Y 'y') Z)
 *
 *	 	(@ members {foo}) + (@ members {bar})	->	(@ members {foobar})
 *
 *	 	(RULES (RULE x y)) + (RULES (RULE z))	->	(RULES (RULE x y z))
 *
 *	 	Rules in root prevent same rule from being appended to RULES node.
 *
 *	 	The goal is a complete combined grammar so we can ignore subordinate
 *	 	grammars.
 */
public void integrateImportedGrammars(Grammar rootGrammar) {
    List<Grammar> imports = rootGrammar.getAllImportedGrammars();
    if (imports == null)
        return;
    GrammarAST root = rootGrammar.ast;
    GrammarAST id = (GrammarAST) root.getChild(0);
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream());
    GrammarAST channelsRoot = (GrammarAST) root.getFirstChildWithType(ANTLRParser.CHANNELS);
    GrammarAST tokensRoot = (GrammarAST) root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
    List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT);
    // Compute list of rules in root grammar and ensure we have a RULES node
    GrammarAST RULES = (GrammarAST) root.getFirstChildWithType(ANTLRParser.RULES);
    Set<String> rootRuleNames = new HashSet<String>();
    // make list of rules we have in root grammar
    List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE);
    for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText());
    // make list of modes we have in root grammar
    List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE);
    Set<String> rootModeNames = new HashSet<String>();
    for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText());
    List<GrammarAST> addedModes = new ArrayList<GrammarAST>();
    for (Grammar imp : imports) {
        // COPY CHANNELS
        GrammarAST imp_channelRoot = (GrammarAST) imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS);
        if (imp_channelRoot != null) {
            rootGrammar.tool.log("grammar", "imported channels: " + imp_channelRoot.getChildren());
            if (channelsRoot == null) {
                channelsRoot = imp_channelRoot.dupTree();
                channelsRoot.g = rootGrammar;
                // ^(GRAMMAR ID TOKENS...)
                root.insertChild(1, channelsRoot);
            } else {
                for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) {
                    String channel = imp_channelRoot.getChild(c).getText();
                    boolean channelIsInRootGrammar = false;
                    for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) {
                        String rootChannel = channelsRoot.getChild(rc).getText();
                        if (rootChannel.equals(channel)) {
                            channelIsInRootGrammar = true;
                            break;
                        }
                    }
                    if (!channelIsInRootGrammar) {
                        channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode());
                    }
                }
            }
        }
        // COPY TOKENS
        GrammarAST imp_tokensRoot = (GrammarAST) imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC);
        if (imp_tokensRoot != null) {
            rootGrammar.tool.log("grammar", "imported tokens: " + imp_tokensRoot.getChildren());
            if (tokensRoot == null) {
                tokensRoot = (GrammarAST) adaptor.create(ANTLRParser.TOKENS_SPEC, "TOKENS");
                tokensRoot.g = rootGrammar;
                // ^(GRAMMAR ID TOKENS...)
                root.insertChild(1, tokensRoot);
            }
            tokensRoot.addChildren(Arrays.asList(imp_tokensRoot.getChildren().toArray(new Tree[0])));
        }
        List<GrammarAST> all_actionRoots = new ArrayList<GrammarAST>();
        List<GrammarAST> imp_actionRoots = imp.ast.getAllChildrenWithType(ANTLRParser.AT);
        if (actionRoots != null)
            all_actionRoots.addAll(actionRoots);
        all_actionRoots.addAll(imp_actionRoots);
        // COPY ACTIONS
        if (imp_actionRoots != null) {
            DoubleKeyMap<String, String, GrammarAST> namedActions = new DoubleKeyMap<String, String, GrammarAST>();
            rootGrammar.tool.log("grammar", "imported actions: " + imp_actionRoots);
            for (GrammarAST at : all_actionRoots) {
                String scopeName = rootGrammar.getDefaultActionScope();
                GrammarAST scope, name, action;
                if (at.getChildCount() > 2) {
                    // must have a scope
                    scope = (GrammarAST) at.getChild(0);
                    scopeName = scope.getText();
                    name = (GrammarAST) at.getChild(1);
                    action = (GrammarAST) at.getChild(2);
                } else {
                    name = (GrammarAST) at.getChild(0);
                    action = (GrammarAST) at.getChild(1);
                }
                GrammarAST prevAction = namedActions.get(scopeName, name.getText());
                if (prevAction == null) {
                    namedActions.put(scopeName, name.getText(), action);
                } else {
                    if (prevAction.g == at.g) {
                        rootGrammar.tool.errMgr.grammarError(ErrorType.ACTION_REDEFINITION, at.g.fileName, name.token, name.getText());
                    } else {
                        String s1 = prevAction.getText();
                        s1 = s1.substring(1, s1.length() - 1);
                        String s2 = action.getText();
                        s2 = s2.substring(1, s2.length() - 1);
                        String combinedAction = "{" + s1 + '\n' + s2 + "}";
                        prevAction.token.setText(combinedAction);
                    }
                }
            }
            // Merge in any actions not in root grammar into root's tree.
            for (String scopeName : namedActions.keySet()) {
                for (String name : namedActions.keySet(scopeName)) {
                    GrammarAST action = namedActions.get(scopeName, name);
                    rootGrammar.tool.log("grammar", action.g.name + " " + scopeName + ":" + name + "=" + action.getText());
                    if (action.g != rootGrammar) {
                        root.insertChild(1, action.getParent());
                    }
                }
            }
        }
        // COPY MODES
        // The strategy is to copy all the mode sections rules across to any
        // mode section in the new grammar with the same name or a new
        // mode section if no matching mode is resolved. Rules which are
        // already in the new grammar are ignored for copy. If the mode
        // section being added ends up empty it is not added to the merged
        // grammar.
        List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE);
        if (modes != null) {
            for (GrammarAST m : modes) {
                rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree());
                String name = m.getChild(0).getText();
                boolean rootAlreadyHasMode = rootModeNames.contains(name);
                GrammarAST destinationAST = null;
                if (rootAlreadyHasMode) {
                    for (GrammarAST m2 : rootModes) {
                        if (m2.getChild(0).getText().equals(name)) {
                            destinationAST = m2;
                            break;
                        }
                    }
                } else {
                    destinationAST = m.dupNode();
                    destinationAST.addChild(m.getChild(0).dupNode());
                }
                int addedRules = 0;
                List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE);
                for (GrammarAST r : modeRules) {
                    rootGrammar.tool.log("grammar", "imported rule: " + r.toStringTree());
                    String ruleName = r.getChild(0).getText();
                    boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName);
                    if (!rootAlreadyHasRule) {
                        destinationAST.addChild(r);
                        addedRules++;
                        rootRuleNames.add(ruleName);
                    }
                }
                if (!rootAlreadyHasMode && addedRules > 0) {
                    rootGrammar.ast.addChild(destinationAST);
                    rootModeNames.add(name);
                    rootModes.add(destinationAST);
                }
            }
        }
        // COPY RULES
        // Rules copied in the mode copy phase are not copied again.
        List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE);
        if (rules != null) {
            for (GrammarAST r : rules) {
                rootGrammar.tool.log("grammar", "imported rule: " + r.toStringTree());
                String name = r.getChild(0).getText();
                boolean rootAlreadyHasRule = rootRuleNames.contains(name);
                if (!rootAlreadyHasRule) {
                    // merge in if not overridden
                    RULES.addChild(r);
                    rootRuleNames.add(name);
                }
            }
        }
        GrammarAST optionsRoot = (GrammarAST) imp.ast.getFirstChildWithType(ANTLRParser.OPTIONS);
        if (optionsRoot != null) {
            // suppress the warning if the options match the options specified
            // in the root grammar
            // https://github.com/antlr/antlr4/issues/707
            boolean hasNewOption = false;
            for (Map.Entry<String, GrammarAST> option : imp.ast.getOptions().entrySet()) {
                String importOption = imp.ast.getOptionString(option.getKey());
                if (importOption == null) {
                    continue;
                }
                String rootOption = rootGrammar.ast.getOptionString(option.getKey());
                if (!importOption.equals(rootOption)) {
                    hasNewOption = true;
                    break;
                }
            }
            if (hasNewOption) {
                rootGrammar.tool.errMgr.grammarError(ErrorType.OPTIONS_IN_DELEGATE, optionsRoot.g.fileName, optionsRoot.token, imp.name);
            }
        }
    }
    rootGrammar.tool.log("grammar", "Grammar: " + rootGrammar.ast.toStringTree());
}

Example 24 with GrammarASTAdaptor

use of org.antlr.v4.parse.GrammarASTAdaptor in project antlr4 by antlr.

the class GrammarTransformPipeline method reduceBlocksToSets.

public void reduceBlocksToSets(GrammarAST root) {
    CommonTreeNodeStream nodes = new CommonTreeNodeStream(new GrammarASTAdaptor(), root);
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
    BlockSetTransformer transformer = new BlockSetTransformer(nodes, g);
    transformer.setTreeAdaptor(adaptor);
    transformer.downup(root);
}

Example 25 with GrammarASTAdaptor

use of org.antlr.v4.parse.GrammarASTAdaptor in project antlr4 by antlr.

the class Grammar method getStringLiteralAliasesFromLexerRules.

/**
 * Return list of (TOKEN_NAME node, 'literal' node) pairs
 */
public static List<Pair<GrammarAST, GrammarAST>> getStringLiteralAliasesFromLexerRules(GrammarRootAST ast) {
    String[] patterns = { "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL)))", "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL ACTION)))", "(RULE %name:TOKEN_REF (BLOCK (ALT %lit:STRING_LITERAL SEMPRED)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) .)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) . .)))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) (LEXER_ACTION_CALL . .))))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) . (LEXER_ACTION_CALL . .))))", "(RULE %name:TOKEN_REF (BLOCK (LEXER_ALT_ACTION (ALT %lit:STRING_LITERAL) (LEXER_ACTION_CALL . .) .)))" // TODO: allow doc comment in there
    };
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(ast.token.getInputStream());
    org.antlr.runtime.tree.TreeWizard wiz = new org.antlr.runtime.tree.TreeWizard(adaptor, ANTLRParser.tokenNames);
    List<Pair<GrammarAST, GrammarAST>> lexerRuleToStringLiteral = new ArrayList<Pair<GrammarAST, GrammarAST>>();
    List<GrammarAST> ruleNodes = ast.getNodesWithType(ANTLRParser.RULE);
    if (ruleNodes == null || ruleNodes.isEmpty())
        return null;
    for (GrammarAST r : ruleNodes) {
        // tool.log("grammar", r.toStringTree());
        // System.out.println("chk: "+r.toStringTree());
        org.antlr.runtime.tree.Tree name = r.getChild(0);
        if (name.getType() == ANTLRParser.TOKEN_REF) {
            // check rule against patterns
            boolean isLitRule;
            for (String pattern : patterns) {
                isLitRule = defAlias(r, pattern, wiz, lexerRuleToStringLiteral);
                if (isLitRule)
                    break;
            }
        // if ( !isLitRule ) System.out.println("no pattern matched");
        }
    }
    return lexerRuleToStringLiteral;
}