Search in sources :

Example 1 with GrammarASTWithOptions

use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by tunnelvisionlabs.

the class GrammarTransformPipeline method augmentTokensWithOriginalPosition.

public static void augmentTokensWithOriginalPosition(final Grammar g, GrammarAST tree) {
    if (tree == null)
        return;
    List<GrammarAST> optionsSubTrees = tree.getNodesWithType(ANTLRParser.ELEMENT_OPTIONS);
    for (int i = 0; i < optionsSubTrees.size(); i++) {
        GrammarAST t = optionsSubTrees.get(i);
        CommonTree elWithOpt = t.parent;
        if (elWithOpt instanceof GrammarASTWithOptions) {
            Map<String, GrammarAST> options = ((GrammarASTWithOptions) elWithOpt).getOptions();
            if (options.containsKey(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME)) {
                GrammarToken newTok = new GrammarToken(g, elWithOpt.getToken());
                newTok.originalTokenIndex = Integer.valueOf(options.get(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME).getText());
                elWithOpt.token = newTok;
                GrammarAST originalNode = g.ast.getNodeWithTokenIndex(newTok.getTokenIndex());
                if (originalNode != null) {
                    // update the AST node start/stop index to match the values
                    // of the corresponding node in the original parse tree.
                    elWithOpt.setTokenStartIndex(originalNode.getTokenStartIndex());
                    elWithOpt.setTokenStopIndex(originalNode.getTokenStopIndex());
                } else {
                    // the original AST node could not be located by index;
                    // make sure to assign valid values for the start/stop
                    // index so toTokenString will not throw exceptions.
                    elWithOpt.setTokenStartIndex(newTok.getTokenIndex());
                    elWithOpt.setTokenStopIndex(newTok.getTokenIndex());
                }
            }
        }
    }
}
Also used : CommonTree(org.antlr.runtime.tree.CommonTree) GrammarAST(org.antlr.v4.tool.ast.GrammarAST) GrammarASTWithOptions(org.antlr.v4.tool.ast.GrammarASTWithOptions) GrammarToken(org.antlr.v4.parse.GrammarToken)

Example 2 with GrammarASTWithOptions

use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by tunnelvisionlabs.

the class ParserATNFactory method _ruleRef.

@NotNull
public Handle _ruleRef(@NotNull GrammarAST node) {
    Rule r = g.getRule(node.getText());
    if (r == null) {
        g.tool.errMgr.grammarError(ErrorType.INTERNAL_ERROR, g.fileName, node.getToken(), "Rule " + node.getText() + " undefined");
        return null;
    }
    RuleStartState start = atn.ruleToStartState[r.index];
    ATNState left = newState(node);
    ATNState right = newState(node);
    int precedence = 0;
    if (((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME) != null) {
        precedence = Integer.parseInt(((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME));
    }
    RuleTransition call = new RuleTransition(start, r.index, precedence, right);
    left.addTransition(call);
    node.atnState = left;
    return new Handle(left, right);
}
Also used : RuleStartState(org.antlr.v4.runtime.atn.RuleStartState) RuleTransition(org.antlr.v4.runtime.atn.RuleTransition) Rule(org.antlr.v4.tool.Rule) LeftRecursiveRule(org.antlr.v4.tool.LeftRecursiveRule) GrammarASTWithOptions(org.antlr.v4.tool.ast.GrammarASTWithOptions) ATNState(org.antlr.v4.runtime.atn.ATNState) NotNull(org.antlr.v4.runtime.misc.NotNull)

Example 3 with GrammarASTWithOptions

use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.

the class ParserATNFactory method _ruleRef.

public Handle _ruleRef(GrammarAST node) {
    Rule r = g.getRule(node.getText());
    if (r == null) {
        g.tool.errMgr.grammarError(ErrorType.INTERNAL_ERROR, g.fileName, node.getToken(), "Rule " + node.getText() + " undefined");
        return null;
    }
    RuleStartState start = atn.ruleToStartState[r.index];
    ATNState left = newState(node);
    ATNState right = newState(node);
    int precedence = 0;
    if (((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME) != null) {
        precedence = Integer.parseInt(((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME));
    }
    RuleTransition call = new RuleTransition(start, r.index, precedence, right);
    left.addTransition(call);
    node.atnState = left;
    return new Handle(left, right);
}
Also used : RuleStartState(org.antlr.v4.runtime.atn.RuleStartState) RuleTransition(org.antlr.v4.runtime.atn.RuleTransition) Rule(org.antlr.v4.tool.Rule) LeftRecursiveRule(org.antlr.v4.tool.LeftRecursiveRule) GrammarASTWithOptions(org.antlr.v4.tool.ast.GrammarASTWithOptions) ATNState(org.antlr.v4.runtime.atn.ATNState)

Example 4 with GrammarASTWithOptions

use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.

the class GrammarTransformPipeline method extractImplicitLexer.

/**
 * Build lexer grammar from combined grammar that looks like:
 *
 *  (COMBINED_GRAMMAR A
 *      (tokens { X (= Y 'y'))
 *      (OPTIONS (= x 'y'))
 *      (@ members {foo})
 *      (@ lexer header {package jj;})
 *      (RULES (RULE .+)))
 *
 *  Move rules and actions to new tree, don't dup. Split AST apart.
 *  We'll have this Grammar share token symbols later; don't generate
 *  tokenVocab or tokens{} section.  Copy over named actions.
 *
 *  Side-effects: it removes children from GRAMMAR &amp; RULES nodes
 *                in combined AST.  Anything cut out is dup'd before
 *                adding to lexer to avoid "who's ur daddy" issues
 */
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
    GrammarRootAST combinedAST = combinedGrammar.ast;
    // tool.log("grammar", "before="+combinedAST.toStringTree());
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
    GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
    // MAKE A GRAMMAR ROOT and ID
    String lexerName = combinedAST.getChild(0).getText() + "Lexer";
    GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
    lexerAST.grammarType = ANTLRParser.LEXER;
    lexerAST.token.setInputStream(combinedAST.token.getInputStream());
    lexerAST.addChild((GrammarAST) adaptor.create(ANTLRParser.ID, lexerName));
    // COPY OPTIONS
    GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
    if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
        GrammarAST lexerOptionsRoot = (GrammarAST) adaptor.dupNode(optionsRoot);
        lexerAST.addChild(lexerOptionsRoot);
        GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
        for (GrammarAST o : options) {
            String optionName = o.getChild(0).getText();
            if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
                GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
                lexerOptionsRoot.addChild(optionTree);
                lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
            }
        }
    }
    // COPY all named actions, but only move those with lexer:: scope
    List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
    for (GrammarAST e : elements) {
        if (e.getType() == ANTLRParser.AT) {
            lexerAST.addChild((Tree) adaptor.dupTree(e));
            if (e.getChild(0).getText().equals("lexer")) {
                actionsWeMoved.add(e);
            }
        }
    }
    for (GrammarAST r : actionsWeMoved) {
        combinedAST.deleteChild(r);
    }
    GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
    if (combinedRulesRoot == null)
        return lexerAST;
    // MOVE lexer rules
    GrammarAST lexerRulesRoot = (GrammarAST) adaptor.create(ANTLRParser.RULES, "RULES");
    lexerAST.addChild(lexerRulesRoot);
    List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
    GrammarASTWithOptions[] rules;
    if (combinedRulesRoot.getChildCount() > 0) {
        rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
    } else {
        rules = new GrammarASTWithOptions[0];
    }
    for (GrammarASTWithOptions r : rules) {
        String ruleName = r.getChild(0).getText();
        if (Grammar.isTokenName(ruleName)) {
            lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
            rulesWeMoved.add(r);
        }
    }
    for (GrammarAST r : rulesWeMoved) {
        combinedRulesRoot.deleteChild(r);
    }
    // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
    List<Pair<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
    Set<String> stringLiterals = combinedGrammar.getStringLiterals();
    // add strings from combined grammar (and imported grammars) into lexer
    // put them first as they are keywords; must resolve ambigs to these rules
    // tool.log("grammar", "strings from parser: "+stringLiterals);
    int insertIndex = 0;
    nextLit: for (String lit : stringLiterals) {
        // if lexer already has a rule for literal, continue
        if (litAliases != null) {
            for (Pair<GrammarAST, GrammarAST> pair : litAliases) {
                GrammarAST litAST = pair.b;
                if (lit.equals(litAST.getText()))
                    continue nextLit;
            }
        }
        // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
        String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
        // can't use wizard; need special node types
        GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
        BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
        AltAST alt = new AltAST(ANTLRParser.ALT);
        TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
        alt.addChild(slit);
        blk.addChild(alt);
        CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
        litRule.addChild(new TerminalAST(idToken));
        litRule.addChild(blk);
        lexerRulesRoot.insertChild(insertIndex, litRule);
        // lexerRulesRoot.getChildren().add(0, litRule);
        // reset indexes and set litRule parent
        lexerRulesRoot.freshenParentAndChildIndexes();
        // next literal will be added after the one just added
        insertIndex++;
    }
    // TODO: take out after stable if slow
    lexerAST.sanityCheckParentAndChildIndexes();
    combinedAST.sanityCheckParentAndChildIndexes();
    // tool.log("grammar", combinedAST.toTokenString());
    combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
    combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
    if (lexerRulesRoot.getChildCount() == 0)
        return null;
    return lexerAST;
}
Also used : RuleAST(org.antlr.v4.tool.ast.RuleAST) GrammarRootAST(org.antlr.v4.tool.ast.GrammarRootAST) GrammarAST(org.antlr.v4.tool.ast.GrammarAST) ArrayList(java.util.ArrayList) BlockAST(org.antlr.v4.tool.ast.BlockAST) AltAST(org.antlr.v4.tool.ast.AltAST) TerminalAST(org.antlr.v4.tool.ast.TerminalAST) GrammarASTAdaptor(org.antlr.v4.parse.GrammarASTAdaptor) CommonToken(org.antlr.runtime.CommonToken) GrammarASTWithOptions(org.antlr.v4.tool.ast.GrammarASTWithOptions) Pair(org.antlr.v4.runtime.misc.Pair)

Example 5 with GrammarASTWithOptions

use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.

the class BasicSemanticChecks method checkElementOptions.

/**
 * Check option is appropriate for elem; parent of ID is ELEMENT_OPTIONS
 */
boolean checkElementOptions(GrammarASTWithOptions elem, GrammarAST ID, GrammarAST valueAST) {
    if (checkAssocElementOption && ID != null && "assoc".equals(ID.getText())) {
        if (elem.getType() != ANTLRParser.ALT) {
            Token optionID = ID.token;
            String fileName = optionID.getInputStream().getSourceName();
            g.tool.errMgr.grammarError(ErrorType.UNRECOGNIZED_ASSOC_OPTION, fileName, optionID, currentRuleName);
        }
    }
    if (elem instanceof RuleRefAST) {
        return checkRuleRefOptions((RuleRefAST) elem, ID, valueAST);
    }
    if (elem instanceof TerminalAST) {
        return checkTokenOptions((TerminalAST) elem, ID, valueAST);
    }
    if (elem.getType() == ANTLRParser.ACTION) {
        return false;
    }
    if (elem.getType() == ANTLRParser.SEMPRED) {
        Token optionID = ID.token;
        String fileName = optionID.getInputStream().getSourceName();
        if (valueAST != null && !Grammar.semPredOptions.contains(optionID.getText())) {
            g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, fileName, optionID, optionID.getText());
            return false;
        }
    }
    return false;
}
Also used : RuleRefAST(org.antlr.v4.tool.ast.RuleRefAST) Token(org.antlr.runtime.Token) TerminalAST(org.antlr.v4.tool.ast.TerminalAST)

Aggregations

GrammarASTWithOptions (org.antlr.v4.tool.ast.GrammarASTWithOptions)11 GrammarAST (org.antlr.v4.tool.ast.GrammarAST)8 ArrayList (java.util.ArrayList)4 CommonToken (org.antlr.runtime.CommonToken)4 Token (org.antlr.runtime.Token)4 GrammarASTAdaptor (org.antlr.v4.parse.GrammarASTAdaptor)4 LeftRecursiveRule (org.antlr.v4.tool.LeftRecursiveRule)4 Rule (org.antlr.v4.tool.Rule)4 TerminalAST (org.antlr.v4.tool.ast.TerminalAST)4 HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 Map (java.util.Map)2 CommonTree (org.antlr.runtime.tree.CommonTree)2 GrammarToken (org.antlr.v4.parse.GrammarToken)2 ATNState (org.antlr.v4.runtime.atn.ATNState)2 RuleStartState (org.antlr.v4.runtime.atn.RuleStartState)2 RuleTransition (org.antlr.v4.runtime.atn.RuleTransition)2 IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)2 AltAST (org.antlr.v4.tool.ast.AltAST)2 BlockAST (org.antlr.v4.tool.ast.BlockAST)2