use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method computeReachSet.
protected SimulatorState computeReachSet(DFA dfa, SimulatorState previous, int t, PredictionContextCache contextCache) {
final boolean useContext = previous.useContext;
ParserRuleContext remainingGlobalContext = previous.remainingOuterContext;
DFAState s = previous.s0;
if (useContext) {
while (s.isContextSymbol(t)) {
DFAState next = null;
if (remainingGlobalContext != null) {
remainingGlobalContext = skipTailCalls(remainingGlobalContext);
next = s.getContextTarget(getReturnState(remainingGlobalContext));
}
if (next == null) {
break;
}
assert remainingGlobalContext != null;
remainingGlobalContext = remainingGlobalContext.getParent();
s = next;
}
}
assert !isAcceptState(s, useContext);
if (isAcceptState(s, useContext)) {
return new SimulatorState(previous.outerContext, s, useContext, remainingGlobalContext);
}
final DFAState s0 = s;
DFAState target = getExistingTargetState(s0, t);
if (target == null) {
Tuple2<DFAState, ParserRuleContext> result = computeTargetState(dfa, s0, remainingGlobalContext, t, useContext, contextCache);
target = result.getItem1();
remainingGlobalContext = result.getItem2();
}
if (target == ERROR) {
return null;
}
assert !useContext || !target.configs.getDipsIntoOuterContext();
return new SimulatorState(previous.outerContext, target, useContext, remainingGlobalContext);
}
use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method computeTargetState.
/**
* Compute a target state for an edge in the DFA, and attempt to add the
* computed state and corresponding edge to the DFA.
*
* @param dfa
* @param s The current DFA state
* @param remainingGlobalContext
* @param t The next input symbol
* @param useContext
* @param contextCache
*
* @return The computed target DFA state for the given input symbol
* {@code t}. If {@code t} does not lead to a valid DFA state, this method
* returns {@link #ERROR}.
*/
@NotNull
protected Tuple2<DFAState, ParserRuleContext> computeTargetState(@NotNull DFA dfa, @NotNull DFAState s, ParserRuleContext remainingGlobalContext, int t, boolean useContext, PredictionContextCache contextCache) {
List<ATNConfig> closureConfigs = new ArrayList<ATNConfig>(s.configs);
IntegerList contextElements = null;
ATNConfigSet reach = new ATNConfigSet();
boolean stepIntoGlobal;
do {
boolean hasMoreContext = !useContext || remainingGlobalContext != null;
if (!hasMoreContext) {
reach.setOutermostConfigSet(true);
}
ATNConfigSet reachIntermediate = new ATNConfigSet();
/* Configurations already in a rule stop state indicate reaching the end
* of the decision rule (local context) or end of the start rule (full
* context). Once reached, these configurations are never updated by a
* closure operation, so they are handled separately for the performance
* advantage of having a smaller intermediate set when calling closure.
*
* For full-context reach operations, separate handling is required to
* ensure that the alternative matching the longest overall sequence is
* chosen when multiple such configurations can match the input.
*/
List<ATNConfig> skippedStopStates = null;
for (ATNConfig c : closureConfigs) {
if (debug)
System.out.println("testing " + getTokenName(t) + " at " + c.toString());
if (c.getState() instanceof RuleStopState) {
assert c.getContext().isEmpty();
if (useContext && !c.getReachesIntoOuterContext() || t == IntStream.EOF) {
if (skippedStopStates == null) {
skippedStopStates = new ArrayList<ATNConfig>();
}
skippedStopStates.add(c);
}
continue;
}
int n = c.getState().getNumberOfOptimizedTransitions();
for (int ti = 0; ti < n; ti++) {
// for each optimized transition
Transition trans = c.getState().getOptimizedTransition(ti);
ATNState target = getReachableTarget(c, trans, t);
if (target != null) {
reachIntermediate.add(c.transform(target, false), contextCache);
}
}
}
/* This block optimizes the reach operation for intermediate sets which
* trivially indicate a termination state for the overall
* adaptivePredict operation.
*
* The conditions assume that intermediate
* contains all configurations relevant to the reach set, but this
* condition is not true when one or more configurations have been
* withheld in skippedStopStates, or when the current symbol is EOF.
*/
if (optimize_unique_closure && skippedStopStates == null && t != Token.EOF && reachIntermediate.getUniqueAlt() != ATN.INVALID_ALT_NUMBER) {
reachIntermediate.setOutermostConfigSet(reach.isOutermostConfigSet());
reach = reachIntermediate;
break;
}
/* If the reach set could not be trivially determined, perform a closure
* operation on the intermediate set to compute its initial value.
*/
final boolean collectPredicates = false;
boolean treatEofAsEpsilon = t == Token.EOF;
closure(reachIntermediate, reach, collectPredicates, hasMoreContext, contextCache, treatEofAsEpsilon);
stepIntoGlobal = reach.getDipsIntoOuterContext();
if (t == IntStream.EOF) {
/* After consuming EOF no additional input is possible, so we are
* only interested in configurations which reached the end of the
* decision rule (local context) or end of the start rule (full
* context). Update reach to contain only these configurations. This
* handles both explicit EOF transitions in the grammar and implicit
* EOF transitions following the end of the decision or start rule.
*
* This is handled before the configurations in skippedStopStates,
* because any configurations potentially added from that list are
* already guaranteed to meet this condition whether or not it's
* required.
*/
reach = removeAllConfigsNotInRuleStopState(reach, contextCache);
}
/* If skippedStopStates is not null, then it contains at least one
* configuration. For full-context reach operations, these
* configurations reached the end of the start rule, in which case we
* only add them back to reach if no configuration during the current
* closure operation reached such a state. This ensures adaptivePredict
* chooses an alternative matching the longest overall sequence when
* multiple alternatives are viable.
*/
if (skippedStopStates != null && (!useContext || !PredictionMode.hasConfigInRuleStopState(reach))) {
assert !skippedStopStates.isEmpty();
for (ATNConfig c : skippedStopStates) {
reach.add(c, contextCache);
}
}
if (useContext && stepIntoGlobal) {
reach.clear();
remainingGlobalContext = skipTailCalls(remainingGlobalContext);
int nextContextElement = getReturnState(remainingGlobalContext);
if (contextElements == null) {
contextElements = new IntegerList();
}
if (remainingGlobalContext.isEmpty()) {
remainingGlobalContext = null;
} else {
remainingGlobalContext = remainingGlobalContext.getParent();
}
contextElements.add(nextContextElement);
if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
for (int i = 0; i < closureConfigs.size(); i++) {
closureConfigs.set(i, closureConfigs.get(i).appendContext(nextContextElement, contextCache));
}
}
}
} while (useContext && stepIntoGlobal);
if (reach.isEmpty()) {
addDFAEdge(s, t, ERROR);
return Tuple.create(ERROR, remainingGlobalContext);
}
DFAState result = addDFAEdge(dfa, s, t, contextElements, reach, contextCache);
return Tuple.create(result, remainingGlobalContext);
}
use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.
the class ATNDeserializer method deserialize.
@SuppressWarnings("deprecation")
public ATN deserialize(@NotNull char[] data) {
data = data.clone();
// was implemented.
for (int i = 1; i < data.length; i++) {
data[i] = (char) (data[i] - 2);
}
int p = 0;
int version = toInt(data[p++]);
if (version != SERIALIZED_VERSION) {
String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with version %d (expected %d).", version, SERIALIZED_VERSION);
throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
}
UUID uuid = toUUID(data, p);
p += 8;
if (!SUPPORTED_UUIDS.contains(uuid)) {
String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with UUID %s (expected %s or a legacy UUID).", uuid, SERIALIZED_UUID);
throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
}
boolean supportsLexerActions = isFeatureSupported(ADDED_LEXER_ACTIONS, uuid);
ATNType grammarType = ATNType.values()[toInt(data[p++])];
int maxTokenType = toInt(data[p++]);
ATN atn = new ATN(grammarType, maxTokenType);
//
// STATES
//
List<Tuple2<LoopEndState, Integer>> loopBackStateNumbers = new ArrayList<Tuple2<LoopEndState, Integer>>();
List<Tuple2<BlockStartState, Integer>> endStateNumbers = new ArrayList<Tuple2<BlockStartState, Integer>>();
int nstates = toInt(data[p++]);
for (int i = 0; i < nstates; i++) {
int stype = toInt(data[p++]);
// ignore bad type of states
if (stype == ATNState.INVALID_TYPE) {
atn.addState(null);
continue;
}
int ruleIndex = toInt(data[p++]);
if (ruleIndex == Character.MAX_VALUE) {
ruleIndex = -1;
}
ATNState s = stateFactory(stype, ruleIndex);
if (stype == ATNState.LOOP_END) {
// special case
int loopBackStateNumber = toInt(data[p++]);
loopBackStateNumbers.add(Tuple.create((LoopEndState) s, loopBackStateNumber));
} else if (s instanceof BlockStartState) {
int endStateNumber = toInt(data[p++]);
endStateNumbers.add(Tuple.create((BlockStartState) s, endStateNumber));
}
atn.addState(s);
}
// delay the assignment of loop back and end states until we know all the state instances have been initialized
for (Tuple2<LoopEndState, Integer> pair : loopBackStateNumbers) {
pair.getItem1().loopBackState = atn.states.get(pair.getItem2());
}
for (Tuple2<BlockStartState, Integer> pair : endStateNumbers) {
pair.getItem1().endState = (BlockEndState) atn.states.get(pair.getItem2());
}
int numNonGreedyStates = toInt(data[p++]);
for (int i = 0; i < numNonGreedyStates; i++) {
int stateNumber = toInt(data[p++]);
((DecisionState) atn.states.get(stateNumber)).nonGreedy = true;
}
int numSllDecisions = toInt(data[p++]);
for (int i = 0; i < numSllDecisions; i++) {
int stateNumber = toInt(data[p++]);
((DecisionState) atn.states.get(stateNumber)).sll = true;
}
int numPrecedenceStates = toInt(data[p++]);
for (int i = 0; i < numPrecedenceStates; i++) {
int stateNumber = toInt(data[p++]);
((RuleStartState) atn.states.get(stateNumber)).isPrecedenceRule = true;
}
//
// RULES
//
int nrules = toInt(data[p++]);
if (atn.grammarType == ATNType.LEXER) {
atn.ruleToTokenType = new int[nrules];
}
atn.ruleToStartState = new RuleStartState[nrules];
for (int i = 0; i < nrules; i++) {
int s = toInt(data[p++]);
RuleStartState startState = (RuleStartState) atn.states.get(s);
startState.leftFactored = toInt(data[p++]) != 0;
atn.ruleToStartState[i] = startState;
if (atn.grammarType == ATNType.LEXER) {
int tokenType = toInt(data[p++]);
if (tokenType == 0xFFFF) {
tokenType = Token.EOF;
}
atn.ruleToTokenType[i] = tokenType;
if (!isFeatureSupported(ADDED_LEXER_ACTIONS, uuid)) {
// this piece of unused metadata was serialized prior to the
// addition of LexerAction
int actionIndexIgnored = toInt(data[p++]);
if (actionIndexIgnored == 0xFFFF) {
actionIndexIgnored = -1;
}
}
}
}
atn.ruleToStopState = new RuleStopState[nrules];
for (ATNState state : atn.states) {
if (!(state instanceof RuleStopState)) {
continue;
}
RuleStopState stopState = (RuleStopState) state;
atn.ruleToStopState[state.ruleIndex] = stopState;
atn.ruleToStartState[state.ruleIndex].stopState = stopState;
}
//
// MODES
//
int nmodes = toInt(data[p++]);
for (int i = 0; i < nmodes; i++) {
int s = toInt(data[p++]);
atn.modeToStartState.add((TokensStartState) atn.states.get(s));
}
atn.modeToDFA = new DFA[nmodes];
for (int i = 0; i < nmodes; i++) {
atn.modeToDFA[i] = new DFA(atn.modeToStartState.get(i));
}
//
// SETS
//
List<IntervalSet> sets = new ArrayList<IntervalSet>();
// First, read all sets with 16-bit Unicode code points <= U+FFFF.
p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP));
// deserialize sets with 32-bit arguments <= U+10FFFF.
if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) {
int previousSetCount = sets.size();
p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP));
atn.setHasUnicodeSMPTransitions(sets.size() > previousSetCount);
}
//
// EDGES
//
int nedges = toInt(data[p++]);
for (int i = 0; i < nedges; i++) {
int src = toInt(data[p]);
int trg = toInt(data[p + 1]);
int ttype = toInt(data[p + 2]);
int arg1 = toInt(data[p + 3]);
int arg2 = toInt(data[p + 4]);
int arg3 = toInt(data[p + 5]);
Transition trans = edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets);
// System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+
// src+"->"+trg+
// " "+Transition.serializationNames[ttype]+
// " "+arg1+","+arg2+","+arg3);
ATNState srcState = atn.states.get(src);
srcState.addTransition(trans);
p += 6;
}
// edges for rule stop states can be derived, so they aren't serialized
// Map rule stop state -> return state -> outermost precedence return
Set<Tuple3<Integer, Integer, Integer>> returnTransitions = new LinkedHashSet<Tuple3<Integer, Integer, Integer>>();
for (ATNState state : atn.states) {
boolean returningToLeftFactored = state.ruleIndex >= 0 && atn.ruleToStartState[state.ruleIndex].leftFactored;
for (int i = 0; i < state.getNumberOfTransitions(); i++) {
Transition t = state.transition(i);
if (!(t instanceof RuleTransition)) {
continue;
}
RuleTransition ruleTransition = (RuleTransition) t;
boolean returningFromLeftFactored = atn.ruleToStartState[ruleTransition.target.ruleIndex].leftFactored;
if (!returningFromLeftFactored && returningToLeftFactored) {
continue;
}
int outermostPrecedenceReturn = -1;
if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isPrecedenceRule) {
if (ruleTransition.precedence == 0) {
outermostPrecedenceReturn = ruleTransition.target.ruleIndex;
}
}
returnTransitions.add(Tuple.create(ruleTransition.target.ruleIndex, ruleTransition.followState.stateNumber, outermostPrecedenceReturn));
}
}
// Add all elements from returnTransitions to the ATN
for (Tuple3<Integer, Integer, Integer> returnTransition : returnTransitions) {
EpsilonTransition transition = new EpsilonTransition(atn.states.get(returnTransition.getItem2()), returnTransition.getItem3());
atn.ruleToStopState[returnTransition.getItem1()].addTransition(transition);
}
for (ATNState state : atn.states) {
if (state instanceof BlockStartState) {
// we need to know the end state to set its start state
if (((BlockStartState) state).endState == null) {
throw new IllegalStateException();
}
// block end states can only be associated to a single block start state
if (((BlockStartState) state).endState.startState != null) {
throw new IllegalStateException();
}
((BlockStartState) state).endState.startState = (BlockStartState) state;
}
if (state instanceof PlusLoopbackState) {
PlusLoopbackState loopbackState = (PlusLoopbackState) state;
for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
ATNState target = loopbackState.transition(i).target;
if (target instanceof PlusBlockStartState) {
((PlusBlockStartState) target).loopBackState = loopbackState;
}
}
} else if (state instanceof StarLoopbackState) {
StarLoopbackState loopbackState = (StarLoopbackState) state;
for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
ATNState target = loopbackState.transition(i).target;
if (target instanceof StarLoopEntryState) {
((StarLoopEntryState) target).loopBackState = loopbackState;
}
}
}
}
//
// DECISIONS
//
int ndecisions = toInt(data[p++]);
for (int i = 1; i <= ndecisions; i++) {
int s = toInt(data[p++]);
DecisionState decState = (DecisionState) atn.states.get(s);
atn.decisionToState.add(decState);
decState.decision = i - 1;
}
//
if (atn.grammarType == ATNType.LEXER) {
if (supportsLexerActions) {
atn.lexerActions = new LexerAction[toInt(data[p++])];
for (int i = 0; i < atn.lexerActions.length; i++) {
LexerActionType actionType = LexerActionType.values()[toInt(data[p++])];
int data1 = toInt(data[p++]);
if (data1 == 0xFFFF) {
data1 = -1;
}
int data2 = toInt(data[p++]);
if (data2 == 0xFFFF) {
data2 = -1;
}
LexerAction lexerAction = lexerActionFactory(actionType, data1, data2);
atn.lexerActions[i] = lexerAction;
}
} else {
// for compatibility with older serialized ATNs, convert the old
// serialized action index for action transitions to the new
// form, which is the index of a LexerCustomAction
List<LexerAction> legacyLexerActions = new ArrayList<LexerAction>();
for (ATNState state : atn.states) {
for (int i = 0; i < state.getNumberOfTransitions(); i++) {
Transition transition = state.transition(i);
if (!(transition instanceof ActionTransition)) {
continue;
}
int ruleIndex = ((ActionTransition) transition).ruleIndex;
int actionIndex = ((ActionTransition) transition).actionIndex;
LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex);
state.setTransition(i, new ActionTransition(transition.target, ruleIndex, legacyLexerActions.size(), false));
legacyLexerActions.add(lexerAction);
}
}
atn.lexerActions = legacyLexerActions.toArray(new LexerAction[legacyLexerActions.size()]);
}
}
markPrecedenceDecisions(atn);
atn.decisionToDFA = new DFA[ndecisions];
for (int i = 0; i < ndecisions; i++) {
atn.decisionToDFA[i] = new DFA(atn.decisionToState.get(i), i);
}
if (deserializationOptions.isVerifyATN()) {
verifyATN(atn);
}
if (deserializationOptions.isGenerateRuleBypassTransitions() && atn.grammarType == ATNType.PARSER) {
atn.ruleToTokenType = new int[atn.ruleToStartState.length];
for (int i = 0; i < atn.ruleToStartState.length; i++) {
atn.ruleToTokenType[i] = atn.maxTokenType + i + 1;
}
for (int i = 0; i < atn.ruleToStartState.length; i++) {
BasicBlockStartState bypassStart = new BasicBlockStartState();
bypassStart.ruleIndex = i;
atn.addState(bypassStart);
BlockEndState bypassStop = new BlockEndState();
bypassStop.ruleIndex = i;
atn.addState(bypassStop);
bypassStart.endState = bypassStop;
atn.defineDecisionState(bypassStart);
bypassStop.startState = bypassStart;
ATNState endState;
Transition excludeTransition = null;
if (atn.ruleToStartState[i].isPrecedenceRule) {
// wrap from the beginning of the rule to the StarLoopEntryState
endState = null;
for (ATNState state : atn.states) {
if (state.ruleIndex != i) {
continue;
}
if (!(state instanceof StarLoopEntryState)) {
continue;
}
ATNState maybeLoopEndState = state.transition(state.getNumberOfTransitions() - 1).target;
if (!(maybeLoopEndState instanceof LoopEndState)) {
continue;
}
if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
endState = state;
break;
}
}
if (endState == null) {
throw new UnsupportedOperationException("Couldn't identify final state of the precedence rule prefix section.");
}
excludeTransition = ((StarLoopEntryState) endState).loopBackState.transition(0);
} else {
endState = atn.ruleToStopState[i];
}
// all non-excluded transitions that currently target end state need to target blockEnd instead
for (ATNState state : atn.states) {
for (Transition transition : state.transitions) {
if (transition == excludeTransition) {
continue;
}
if (transition.target == endState) {
transition.target = bypassStop;
}
}
}
// all transitions leaving the rule start state need to leave blockStart instead
while (atn.ruleToStartState[i].getNumberOfTransitions() > 0) {
Transition transition = atn.ruleToStartState[i].removeTransition(atn.ruleToStartState[i].getNumberOfTransitions() - 1);
bypassStart.addTransition(transition);
}
// link the new states
atn.ruleToStartState[i].addTransition(new EpsilonTransition(bypassStart));
bypassStop.addTransition(new EpsilonTransition(endState));
ATNState matchState = new BasicState();
atn.addState(matchState);
matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i]));
bypassStart.addTransition(new EpsilonTransition(matchState));
}
if (deserializationOptions.isVerifyATN()) {
// reverify after modification
verifyATN(atn);
}
}
if (deserializationOptions.isOptimize()) {
while (true) {
int optimizationCount = 0;
optimizationCount += inlineSetRules(atn);
optimizationCount += combineChainedEpsilons(atn);
boolean preserveOrder = atn.grammarType == ATNType.LEXER;
optimizationCount += optimizeSets(atn, preserveOrder);
if (optimizationCount == 0) {
break;
}
}
if (deserializationOptions.isVerifyATN()) {
// reverify after modification
verifyATN(atn);
}
}
identifyTailCalls(atn);
return atn;
}
use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.
the class ParserInterpreter method parse.
/**
* Begin parsing at startRuleIndex
*/
public ParserRuleContext parse(int startRuleIndex) {
RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex];
rootContext = createInterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
if (startRuleStartState.isPrecedenceRule) {
enterRecursionRule(rootContext, startRuleStartState.stateNumber, startRuleIndex, 0);
} else {
enterRule(rootContext, startRuleStartState.stateNumber, startRuleIndex);
}
while (true) {
ATNState p = getATNState();
switch(p.getStateType()) {
case ATNState.RULE_STOP:
// pop; return from rule
if (_ctx.isEmpty()) {
if (startRuleStartState.isPrecedenceRule) {
ParserRuleContext result = _ctx;
Tuple2<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
unrollRecursionContexts(parentContext.getItem1());
return result;
} else {
exitRule();
return rootContext;
}
}
visitRuleStopState(p);
break;
default:
try {
visitState(p);
} catch (RecognitionException e) {
setState(atn.ruleToStopState[p.ruleIndex].stateNumber);
getContext().exception = e;
getErrorHandler().reportError(this, e);
recover(e);
}
break;
}
}
}
use of org.antlr.v4.runtime.misc.Tuple2 in project antlr4 by tunnelvisionlabs.
the class ParserInterpreter method visitRuleStopState.
protected void visitRuleStopState(ATNState p) {
RuleStartState ruleStartState = atn.ruleToStartState[p.ruleIndex];
if (ruleStartState.isPrecedenceRule) {
Tuple2<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
unrollRecursionContexts(parentContext.getItem1());
setState(parentContext.getItem2());
} else {
exitRule();
}
RuleTransition ruleTransition = (RuleTransition) atn.states.get(getState()).transition(0);
setState(ruleTransition.followState.stateNumber);
}
Aggregations