use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by tunnelvisionlabs.
the class GrammarTransformPipeline method augmentTokensWithOriginalPosition.
public static void augmentTokensWithOriginalPosition(final Grammar g, GrammarAST tree) {
if (tree == null)
return;
List<GrammarAST> optionsSubTrees = tree.getNodesWithType(ANTLRParser.ELEMENT_OPTIONS);
for (int i = 0; i < optionsSubTrees.size(); i++) {
GrammarAST t = optionsSubTrees.get(i);
CommonTree elWithOpt = t.parent;
if (elWithOpt instanceof GrammarASTWithOptions) {
Map<String, GrammarAST> options = ((GrammarASTWithOptions) elWithOpt).getOptions();
if (options.containsKey(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME)) {
GrammarToken newTok = new GrammarToken(g, elWithOpt.getToken());
newTok.originalTokenIndex = Integer.valueOf(options.get(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME).getText());
elWithOpt.token = newTok;
GrammarAST originalNode = g.ast.getNodeWithTokenIndex(newTok.getTokenIndex());
if (originalNode != null) {
// update the AST node start/stop index to match the values
// of the corresponding node in the original parse tree.
elWithOpt.setTokenStartIndex(originalNode.getTokenStartIndex());
elWithOpt.setTokenStopIndex(originalNode.getTokenStopIndex());
} else {
// the original AST node could not be located by index;
// make sure to assign valid values for the start/stop
// index so toTokenString will not throw exceptions.
elWithOpt.setTokenStartIndex(newTok.getTokenIndex());
elWithOpt.setTokenStopIndex(newTok.getTokenIndex());
}
}
}
}
}
use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by tunnelvisionlabs.
the class ParserATNFactory method _ruleRef.
@NotNull
public Handle _ruleRef(@NotNull GrammarAST node) {
Rule r = g.getRule(node.getText());
if (r == null) {
g.tool.errMgr.grammarError(ErrorType.INTERNAL_ERROR, g.fileName, node.getToken(), "Rule " + node.getText() + " undefined");
return null;
}
RuleStartState start = atn.ruleToStartState[r.index];
ATNState left = newState(node);
ATNState right = newState(node);
int precedence = 0;
if (((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME) != null) {
precedence = Integer.parseInt(((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME));
}
RuleTransition call = new RuleTransition(start, r.index, precedence, right);
left.addTransition(call);
node.atnState = left;
return new Handle(left, right);
}
use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.
the class ParserATNFactory method _ruleRef.
public Handle _ruleRef(GrammarAST node) {
Rule r = g.getRule(node.getText());
if (r == null) {
g.tool.errMgr.grammarError(ErrorType.INTERNAL_ERROR, g.fileName, node.getToken(), "Rule " + node.getText() + " undefined");
return null;
}
RuleStartState start = atn.ruleToStartState[r.index];
ATNState left = newState(node);
ATNState right = newState(node);
int precedence = 0;
if (((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME) != null) {
precedence = Integer.parseInt(((GrammarASTWithOptions) node).getOptionString(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME));
}
RuleTransition call = new RuleTransition(start, r.index, precedence, right);
left.addTransition(call);
node.atnState = left;
return new Handle(left, right);
}
use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.
the class GrammarTransformPipeline method extractImplicitLexer.
/**
* Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
GrammarRootAST combinedAST = combinedGrammar.ast;
// tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
// MAKE A GRAMMAR ROOT and ID
String lexerName = combinedAST.getChild(0).getText() + "Lexer";
GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.token.setInputStream(combinedAST.token.getInputStream());
lexerAST.addChild((GrammarAST) adaptor.create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
GrammarAST lexerOptionsRoot = (GrammarAST) adaptor.dupNode(optionsRoot);
lexerAST.addChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
for (GrammarAST o : options) {
String optionName = o.getChild(0).getText();
if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
lexerOptionsRoot.addChild(optionTree);
lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
for (GrammarAST e : elements) {
if (e.getType() == ANTLRParser.AT) {
lexerAST.addChild((Tree) adaptor.dupTree(e));
if (e.getChild(0).getText().equals("lexer")) {
actionsWeMoved.add(e);
}
}
}
for (GrammarAST r : actionsWeMoved) {
combinedAST.deleteChild(r);
}
GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
return lexerAST;
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST) adaptor.create(ANTLRParser.RULES, "RULES");
lexerAST.addChild(lexerRulesRoot);
List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.getChildCount() > 0) {
rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
} else {
rules = new GrammarASTWithOptions[0];
}
for (GrammarASTWithOptions r : rules) {
String ruleName = r.getChild(0).getText();
if (Grammar.isTokenName(ruleName)) {
lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
rulesWeMoved.add(r);
}
}
for (GrammarAST r : rulesWeMoved) {
combinedRulesRoot.deleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
List<Pair<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
Set<String> stringLiterals = combinedGrammar.getStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
nextLit: for (String lit : stringLiterals) {
// if lexer already has a rule for literal, continue
if (litAliases != null) {
for (Pair<GrammarAST, GrammarAST> pair : litAliases) {
GrammarAST litAST = pair.b;
if (lit.equals(litAST.getText()))
continue nextLit;
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.addChild(slit);
blk.addChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.addChild(new TerminalAST(idToken));
litRule.addChild(blk);
lexerRulesRoot.insertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
// reset indexes and set litRule parent
lexerRulesRoot.freshenParentAndChildIndexes();
// next literal will be added after the one just added
insertIndex++;
}
// TODO: take out after stable if slow
lexerAST.sanityCheckParentAndChildIndexes();
combinedAST.sanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
if (lexerRulesRoot.getChildCount() == 0)
return null;
return lexerAST;
}
use of org.antlr.v4.tool.ast.GrammarASTWithOptions in project antlr4 by antlr.
the class BasicSemanticChecks method checkElementOptions.
/**
* Check option is appropriate for elem; parent of ID is ELEMENT_OPTIONS
*/
boolean checkElementOptions(GrammarASTWithOptions elem, GrammarAST ID, GrammarAST valueAST) {
if (checkAssocElementOption && ID != null && "assoc".equals(ID.getText())) {
if (elem.getType() != ANTLRParser.ALT) {
Token optionID = ID.token;
String fileName = optionID.getInputStream().getSourceName();
g.tool.errMgr.grammarError(ErrorType.UNRECOGNIZED_ASSOC_OPTION, fileName, optionID, currentRuleName);
}
}
if (elem instanceof RuleRefAST) {
return checkRuleRefOptions((RuleRefAST) elem, ID, valueAST);
}
if (elem instanceof TerminalAST) {
return checkTokenOptions((TerminalAST) elem, ID, valueAST);
}
if (elem.getType() == ANTLRParser.ACTION) {
return false;
}
if (elem.getType() == ANTLRParser.SEMPRED) {
Token optionID = ID.token;
String fileName = optionID.getInputStream().getSourceName();
if (valueAST != null && !Grammar.semPredOptions.contains(optionID.getText())) {
g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, fileName, optionID, optionID.getText());
return false;
}
}
return false;
}
Aggregations