use of org.antlr.v4.runtime.atn.Transition in project antlr4 by tunnelvisionlabs.
the class TestAmbigParseTrees method testInterpAtSpecificAlt.
void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g, String startRule, int startAlt, String input, String expectedParseTree) {
LexerInterpreter lexEngine = lg.createLexerInterpreter(CharStreams.fromString(input));
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
Transition tr = ruleStartState.transition(0);
ATNState t2 = tr.target;
if (!(t2 instanceof BasicBlockStartState)) {
throw new IllegalArgumentException("rule has no decision: " + startRule);
}
parser.addDecisionOverride(((DecisionState) t2).decision, 0, startAlt);
ParseTree t = parser.parse(g.rules.get(startRule).index);
InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
}
use of org.antlr.v4.runtime.atn.Transition in project antlr4 by tunnelvisionlabs.
the class ATNOptimizer method optimizeSets.
private static void optimizeSets(Grammar g, ATN atn) {
if (g.isParser()) {
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
List<DecisionState> decisions = atn.decisionToState;
for (DecisionState decision : decisions) {
if (decision.ruleIndex >= 0) {
Rule rule = g.getRule(decision.ruleIndex);
if (Character.isLowerCase(rule.name.charAt(0))) {
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.getNumberOfTransitions(); i++) {
Transition epsTransition = decision.transition(i);
if (!(epsTransition instanceof EpsilonTransition)) {
continue;
}
if (epsTransition.target.getNumberOfTransitions() != 1) {
continue;
}
Transition transition = epsTransition.target.transition(0);
if (!(transition.target instanceof BlockEndState)) {
continue;
}
if (transition instanceof NotSetTransition) {
// TODO: not yet implemented
continue;
}
if (transition instanceof AtomTransition || transition instanceof RangeTransition || transition instanceof SetTransition) {
setTransitions.add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.getIntervals().size() - 1; i >= 0; i--) {
Interval interval = setTransitions.getIntervals().get(i);
if (interval.length() <= 1) {
continue;
}
ATNState blockEndState = decision.transition(interval.a).target.transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++) {
Transition matchTransition = decision.transition(j).target.transition(0);
if (matchTransition instanceof NotSetTransition) {
throw new UnsupportedOperationException("Not yet implemented.");
}
IntervalSet set = matchTransition.label();
List<Interval> intervals = set.getIntervals();
int n = intervals.size();
for (int k = 0; k < n; k++) {
Interval setInterval = intervals.get(k);
int a = setInterval.a;
int b = setInterval.b;
if (a != -1 && b != -1) {
for (int v = a; v <= b; v++) {
if (matchSet.contains(v)) {
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, null, CharSupport.getANTLRCharLiteralForChar(v), CharSupport.getIntervalSetEscapedString(matchSet));
break;
}
}
}
}
matchSet.addAll(set);
}
Transition newTransition;
if (matchSet.getIntervals().size() == 1) {
if (matchSet.size() == 1) {
newTransition = CodePointTransitions.createWithCodePoint(blockEndState, matchSet.getMinElement());
} else {
Interval matchInterval = matchSet.getIntervals().get(0);
newTransition = CodePointTransitions.createWithCodePointRange(blockEndState, matchInterval.a, matchInterval.b);
}
} else {
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.transition(interval.a).target.setTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++) {
Transition removed = decision.removeTransition(interval.a + 1);
atn.removeState(removed.target);
removedStates++;
}
}
}
// System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
use of org.antlr.v4.runtime.atn.Transition in project antlr4 by tunnelvisionlabs.
the class LexerATNFactory method stringLiteral.
/**
* For a lexer, a string is a sequence of char to match. That is,
* "fog" is treated as 'f' 'o' 'g' not as a single transition in
* the DFA. Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states
* for n characters.
*/
@Override
public Handle stringLiteral(TerminalAST stringLiteralAST) {
String chars = stringLiteralAST.getText();
ATNState left = newState(stringLiteralAST);
ATNState right;
String s = CharSupport.getStringFromGrammarStringLiteral(chars);
if (s == null) {
// the lexer will already have given an error
return new Handle(left, left);
}
int n = s.length();
ATNState prev = left;
right = null;
for (int i = 0; i < n; ) {
right = newState(stringLiteralAST);
int codePoint = s.codePointAt(i);
prev.addTransition(CodePointTransitions.createWithCodePoint(right, codePoint));
prev = right;
i += Character.charCount(codePoint);
}
stringLiteralAST.atnState = left;
return new Handle(left, right);
}
use of org.antlr.v4.runtime.atn.Transition in project antlr4 by tunnelvisionlabs.
the class LexerATNFactory method tokenRef.
@Override
public Handle tokenRef(TerminalAST node) {
// Ref to EOF in lexer yields char transition on -1
if (node.getText().equals("EOF")) {
ATNState left = newState(node);
ATNState right = newState(node);
left.addTransition(new AtomTransition(right, IntStream.EOF));
return new Handle(left, right);
}
return _ruleRef(node);
}
use of org.antlr.v4.runtime.atn.Transition in project antlr4 by tunnelvisionlabs.
the class LexerATNFactory method set.
@Override
public Handle set(GrammarAST associatedAST, List<GrammarAST> alts, boolean invert) {
ATNState left = newState(associatedAST);
ATNState right = newState(associatedAST);
IntervalSet set = new IntervalSet();
for (GrammarAST t : alts) {
if (t.getType() == ANTLRParser.RANGE) {
int a = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(0).getText());
int b = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(1).getText());
if (checkRange((GrammarAST) t.getChild(0), (GrammarAST) t.getChild(1), a, b)) {
checkSetCollision(associatedAST, set, a, b);
set.add(a, b);
}
} else if (t.getType() == ANTLRParser.LEXER_CHAR_SET) {
set.addAll(getSetFromCharSetLiteral(t));
} else if (t.getType() == ANTLRParser.STRING_LITERAL) {
int c = CharSupport.getCharValueFromGrammarCharLiteral(t.getText());
if (c != -1) {
checkSetCollision(associatedAST, set, c);
set.add(c);
} else {
g.tool.errMgr.grammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET, g.fileName, t.getToken(), t.getText());
}
} else if (t.getType() == ANTLRParser.TOKEN_REF) {
g.tool.errMgr.grammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET, g.fileName, t.getToken(), t.getText());
}
}
if (invert) {
left.addTransition(new NotSetTransition(right, set));
} else {
Transition transition;
if (set.getIntervals().size() == 1) {
Interval interval = set.getIntervals().get(0);
transition = CodePointTransitions.createWithCodePointRange(right, interval.a, interval.b);
} else {
transition = new SetTransition(right, set);
}
left.addTransition(transition);
}
associatedAST.atnState = left;
return new Handle(left, right);
}
Aggregations