Search in sources :

Example 1 with RuleAST

use of org.antlr.v4.tool.ast.RuleAST in project antlr4 by antlr.

the class Grammar method getStateToGrammarRegionMap.

public static Map<Integer, Interval> getStateToGrammarRegionMap(GrammarRootAST ast, IntervalSet grammarTokenTypes) {
    Map<Integer, Interval> stateToGrammarRegionMap = new HashMap<Integer, Interval>();
    if (ast == null)
        return stateToGrammarRegionMap;
    List<GrammarAST> nodes = ast.getNodesWithType(grammarTokenTypes);
    for (GrammarAST n : nodes) {
        if (n.atnState != null) {
            Interval tokenRegion = Interval.of(n.getTokenStartIndex(), n.getTokenStopIndex());
            org.antlr.runtime.tree.Tree ruleNode = null;
            // RULEs, BLOCKs of transformed recursive rules point to original token interval
            switch(n.getType()) {
                case ANTLRParser.RULE:
                    ruleNode = n;
                    break;
                case ANTLRParser.BLOCK:
                case ANTLRParser.CLOSURE:
                    ruleNode = n.getAncestor(ANTLRParser.RULE);
                    break;
            }
            if (ruleNode instanceof RuleAST) {
                String ruleName = ((RuleAST) ruleNode).getRuleName();
                Rule r = ast.g.getRule(ruleName);
                if (r instanceof LeftRecursiveRule) {
                    RuleAST originalAST = ((LeftRecursiveRule) r).getOriginalAST();
                    tokenRegion = Interval.of(originalAST.getTokenStartIndex(), originalAST.getTokenStopIndex());
                }
            }
            stateToGrammarRegionMap.put(n.atnState.stateNumber, tokenRegion);
        }
    }
    return stateToGrammarRegionMap;
}
Also used : RuleAST(org.antlr.v4.tool.ast.RuleAST) OrderedHashMap(org.antlr.v4.misc.OrderedHashMap) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) GrammarAST(org.antlr.v4.tool.ast.GrammarAST) Interval(org.antlr.v4.runtime.misc.Interval)

Example 2 with RuleAST

use of org.antlr.v4.tool.ast.RuleAST in project antlr4 by antlr.

the class GrammarTransformPipeline method extractImplicitLexer.

/** Build lexer grammar from combined grammar that looks like:
	 *
	 *  (COMBINED_GRAMMAR A
	 *      (tokens { X (= Y 'y'))
	 *      (OPTIONS (= x 'y'))
	 *      (@ members {foo})
	 *      (@ lexer header {package jj;})
	 *      (RULES (RULE .+)))
	 *
	 *  Move rules and actions to new tree, don't dup. Split AST apart.
	 *  We'll have this Grammar share token symbols later; don't generate
	 *  tokenVocab or tokens{} section.  Copy over named actions.
	 *
	 *  Side-effects: it removes children from GRAMMAR &amp; RULES nodes
	 *                in combined AST.  Anything cut out is dup'd before
	 *                adding to lexer to avoid "who's ur daddy" issues
	 */
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
    GrammarRootAST combinedAST = combinedGrammar.ast;
    //tool.log("grammar", "before="+combinedAST.toStringTree());
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
    GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
    // MAKE A GRAMMAR ROOT and ID
    String lexerName = combinedAST.getChild(0).getText() + "Lexer";
    GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
    lexerAST.grammarType = ANTLRParser.LEXER;
    lexerAST.token.setInputStream(combinedAST.token.getInputStream());
    lexerAST.addChild((GrammarAST) adaptor.create(ANTLRParser.ID, lexerName));
    // COPY OPTIONS
    GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
    if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
        GrammarAST lexerOptionsRoot = (GrammarAST) adaptor.dupNode(optionsRoot);
        lexerAST.addChild(lexerOptionsRoot);
        GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
        for (GrammarAST o : options) {
            String optionName = o.getChild(0).getText();
            if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
                GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
                lexerOptionsRoot.addChild(optionTree);
                lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
            }
        }
    }
    // COPY all named actions, but only move those with lexer:: scope
    List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
    for (GrammarAST e : elements) {
        if (e.getType() == ANTLRParser.AT) {
            lexerAST.addChild((Tree) adaptor.dupTree(e));
            if (e.getChild(0).getText().equals("lexer")) {
                actionsWeMoved.add(e);
            }
        }
    }
    for (GrammarAST r : actionsWeMoved) {
        combinedAST.deleteChild(r);
    }
    GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
    if (combinedRulesRoot == null)
        return lexerAST;
    // MOVE lexer rules
    GrammarAST lexerRulesRoot = (GrammarAST) adaptor.create(ANTLRParser.RULES, "RULES");
    lexerAST.addChild(lexerRulesRoot);
    List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
    GrammarASTWithOptions[] rules;
    if (combinedRulesRoot.getChildCount() > 0) {
        rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
    } else {
        rules = new GrammarASTWithOptions[0];
    }
    for (GrammarASTWithOptions r : rules) {
        String ruleName = r.getChild(0).getText();
        if (Grammar.isTokenName(ruleName)) {
            lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
            rulesWeMoved.add(r);
        }
    }
    for (GrammarAST r : rulesWeMoved) {
        combinedRulesRoot.deleteChild(r);
    }
    // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
    List<Pair<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
    Set<String> stringLiterals = combinedGrammar.getStringLiterals();
    // add strings from combined grammar (and imported grammars) into lexer
    // put them first as they are keywords; must resolve ambigs to these rules
    //		tool.log("grammar", "strings from parser: "+stringLiterals);
    int insertIndex = 0;
    nextLit: for (String lit : stringLiterals) {
        // if lexer already has a rule for literal, continue
        if (litAliases != null) {
            for (Pair<GrammarAST, GrammarAST> pair : litAliases) {
                GrammarAST litAST = pair.b;
                if (lit.equals(litAST.getText()))
                    continue nextLit;
            }
        }
        // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
        String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
        // can't use wizard; need special node types
        GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
        BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
        AltAST alt = new AltAST(ANTLRParser.ALT);
        TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
        alt.addChild(slit);
        blk.addChild(alt);
        CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
        litRule.addChild(new TerminalAST(idToken));
        litRule.addChild(blk);
        lexerRulesRoot.insertChild(insertIndex, litRule);
        //			lexerRulesRoot.getChildren().add(0, litRule);
        // reset indexes and set litRule parent
        lexerRulesRoot.freshenParentAndChildIndexes();
        // next literal will be added after the one just added
        insertIndex++;
    }
    // TODO: take out after stable if slow
    lexerAST.sanityCheckParentAndChildIndexes();
    combinedAST.sanityCheckParentAndChildIndexes();
    //		tool.log("grammar", combinedAST.toTokenString());
    combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
    combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
    if (lexerRulesRoot.getChildCount() == 0)
        return null;
    return lexerAST;
}
Also used : RuleAST(org.antlr.v4.tool.ast.RuleAST) GrammarRootAST(org.antlr.v4.tool.ast.GrammarRootAST) GrammarAST(org.antlr.v4.tool.ast.GrammarAST) ArrayList(java.util.ArrayList) BlockAST(org.antlr.v4.tool.ast.BlockAST) AltAST(org.antlr.v4.tool.ast.AltAST) TerminalAST(org.antlr.v4.tool.ast.TerminalAST) GrammarASTAdaptor(org.antlr.v4.parse.GrammarASTAdaptor) CommonToken(org.antlr.runtime.CommonToken) GrammarASTWithOptions(org.antlr.v4.tool.ast.GrammarASTWithOptions) Pair(org.antlr.v4.runtime.misc.Pair)

Example 3 with RuleAST

use of org.antlr.v4.tool.ast.RuleAST in project antlr4 by antlr.

the class GrammarASTAdaptor method create.

@Override
public /** Make sure even imaginary nodes know the input stream */
Object create(int tokenType, String text) {
    GrammarAST t;
    if (tokenType == ANTLRParser.RULE) {
        // needed by TreeWizard to make RULE tree
        t = new RuleAST(new CommonToken(tokenType, text));
    } else if (tokenType == ANTLRParser.STRING_LITERAL) {
        // implicit lexer construction done with wizard; needs this node type
        // whereas grammar ANTLRParser.g can use token option to spec node type
        t = new TerminalAST(new CommonToken(tokenType, text));
    } else {
        t = (GrammarAST) super.create(tokenType, text);
    }
    t.token.setInputStream(input);
    return t;
}
Also used : RuleAST(org.antlr.v4.tool.ast.RuleAST) GrammarAST(org.antlr.v4.tool.ast.GrammarAST) CommonToken(org.antlr.runtime.CommonToken) TerminalAST(org.antlr.v4.tool.ast.TerminalAST)

Example 4 with RuleAST

use of org.antlr.v4.tool.ast.RuleAST in project antlr4 by antlr.

the class BasicSemanticChecks method finishRule.

@Override
public void finishRule(RuleAST rule, GrammarAST ID, GrammarAST block) {
    if (rule.isLexerRule())
        return;
    BlockAST blk = (BlockAST) rule.getFirstChildWithType(BLOCK);
    int nalts = blk.getChildCount();
    GrammarAST idAST = (GrammarAST) rule.getChild(0);
    for (int i = 0; i < nalts; i++) {
        AltAST altAST = (AltAST) blk.getChild(i);
        if (altAST.altLabel != null) {
            String altLabel = altAST.altLabel.getText();
            // first check that label doesn't conflict with a rule
            // label X or x can't be rule x.
            Rule r = ruleCollector.rules.get(Utils.decapitalize(altLabel));
            if (r != null) {
                g.tool.errMgr.grammarError(ErrorType.ALT_LABEL_CONFLICTS_WITH_RULE, g.fileName, altAST.altLabel.token, altLabel, r.name);
            }
            // Now verify that label X or x doesn't conflict with label
            // in another rule. altLabelToRuleName has both X and x mapped.
            String prevRuleForLabel = ruleCollector.altLabelToRuleName.get(altLabel);
            if (prevRuleForLabel != null && !prevRuleForLabel.equals(rule.getRuleName())) {
                g.tool.errMgr.grammarError(ErrorType.ALT_LABEL_REDEF, g.fileName, altAST.altLabel.token, altLabel, rule.getRuleName(), prevRuleForLabel);
            }
        }
    }
    List<GrammarAST> altLabels = ruleCollector.ruleToAltLabels.get(rule.getRuleName());
    int numAltLabels = 0;
    if (altLabels != null)
        numAltLabels = altLabels.size();
    if (numAltLabels > 0 && nalts != numAltLabels) {
        g.tool.errMgr.grammarError(ErrorType.RULE_WITH_TOO_FEW_ALT_LABELS, g.fileName, idAST.token, rule.getRuleName());
    }
}
Also used : GrammarAST(org.antlr.v4.tool.ast.GrammarAST) BlockAST(org.antlr.v4.tool.ast.BlockAST) AltAST(org.antlr.v4.tool.ast.AltAST) Rule(org.antlr.v4.tool.Rule)

Example 5 with RuleAST

use of org.antlr.v4.tool.ast.RuleAST in project antlr4 by antlr.

the class ParserATNFactory method rule.

/* start->ruleblock->end */
@Override
public Handle rule(GrammarAST ruleAST, String name, Handle blk) {
    Rule r = g.getRule(name);
    RuleStartState start = atn.ruleToStartState[r.index];
    epsilon(start, blk.left);
    RuleStopState stop = atn.ruleToStopState[r.index];
    epsilon(blk.right, stop);
    Handle h = new Handle(start, stop);
    //		ATNPrinter ser = new ATNPrinter(g, h.left);
    //		System.out.println(ruleAST.toStringTree()+":\n"+ser.asString());
    ruleAST.atnState = start;
    return h;
}
Also used : RuleStopState(org.antlr.v4.runtime.atn.RuleStopState) RuleStartState(org.antlr.v4.runtime.atn.RuleStartState) Rule(org.antlr.v4.tool.Rule) LeftRecursiveRule(org.antlr.v4.tool.LeftRecursiveRule)

Aggregations

GrammarAST (org.antlr.v4.tool.ast.GrammarAST)8 RuleAST (org.antlr.v4.tool.ast.RuleAST)7 Rule (org.antlr.v4.tool.Rule)4 LeftRecursiveRule (org.antlr.v4.tool.LeftRecursiveRule)3 ActionAST (org.antlr.v4.tool.ast.ActionAST)3 AltAST (org.antlr.v4.tool.ast.AltAST)3 BlockAST (org.antlr.v4.tool.ast.BlockAST)3 TerminalAST (org.antlr.v4.tool.ast.TerminalAST)3 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 CommonToken (org.antlr.runtime.CommonToken)2 RecognitionException (org.antlr.runtime.RecognitionException)2 GrammarASTAdaptor (org.antlr.v4.parse.GrammarASTAdaptor)2 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)1 ANTLRStringStream (org.antlr.runtime.ANTLRStringStream)1 CommonTokenStream (org.antlr.runtime.CommonTokenStream)1 ParserRuleReturnScope (org.antlr.runtime.ParserRuleReturnScope)1 Token (org.antlr.runtime.Token)1 OrderedHashMap (org.antlr.v4.misc.OrderedHashMap)1