use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method getStartState.
protected SimulatorState getStartState(@NotNull DFA dfa, @NotNull TokenStream input, @NotNull ParserRuleContext outerContext, boolean useContext) {
if (!useContext) {
if (dfa.isPrecedenceDfa()) {
// the start state for a precedence DFA depends on the current
// parser precedence, and is provided by a DFA method.
DFAState state = dfa.getPrecedenceStartState(parser.getPrecedence(), false);
if (state == null) {
return null;
}
return new SimulatorState(outerContext, state, false, outerContext);
} else {
if (dfa.s0.get() == null) {
return null;
}
return new SimulatorState(outerContext, dfa.s0.get(), false, outerContext);
}
}
if (!enable_global_context_dfa) {
return null;
}
ParserRuleContext remainingContext = outerContext;
assert outerContext != null;
DFAState s0;
if (dfa.isPrecedenceDfa()) {
s0 = dfa.getPrecedenceStartState(parser.getPrecedence(), true);
} else {
s0 = dfa.s0full.get();
}
while (remainingContext != null && s0 != null && s0.isContextSensitive()) {
remainingContext = skipTailCalls(remainingContext);
s0 = s0.getContextTarget(getReturnState(remainingContext));
if (remainingContext.isEmpty()) {
assert s0 == null || !s0.isContextSensitive();
} else {
remainingContext = remainingContext.getParent();
}
}
if (s0 == null) {
return null;
}
return new SimulatorState(outerContext, s0, useContext, remainingContext);
}
use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method execDFA.
protected int execDFA(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState state) {
ParserRuleContext outerContext = state.outerContext;
if (dfa_debug)
System.out.println("DFA decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input) + ", outerContext=" + outerContext.toString(parser));
if (dfa_debug)
System.out.print(dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
DFAState s = state.s0;
int t = input.LA(1);
ParserRuleContext remainingOuterContext = state.remainingOuterContext;
while (true) {
if (dfa_debug)
System.out.println("DFA state " + s.stateNumber + " LA(1)==" + getLookaheadName(input));
if (state.useContext) {
while (s.isContextSymbol(t)) {
DFAState next = null;
if (remainingOuterContext != null) {
remainingOuterContext = skipTailCalls(remainingOuterContext);
next = s.getContextTarget(getReturnState(remainingOuterContext));
}
if (next == null) {
// fail over to ATN
SimulatorState initialState = new SimulatorState(state.outerContext, s, state.useContext, remainingOuterContext);
return execATN(dfa, input, startIndex, initialState);
}
assert remainingOuterContext != null;
remainingOuterContext = remainingOuterContext.getParent();
s = next;
}
}
if (isAcceptState(s, state.useContext)) {
if (s.predicates != null) {
if (dfa_debug)
System.out.println("accept " + s);
} else {
if (dfa_debug)
System.out.println("accept; predict " + s.getPrediction() + " in state " + s.stateNumber);
}
// TODO: v3 dfa don't do this.
break;
}
// t is not updated if one of these states is reached
assert !isAcceptState(s, state.useContext);
// if no edge, pop over to ATN interpreter, update DFA and return
DFAState target = getExistingTargetState(s, t);
if (target == null) {
if (dfa_debug && t >= 0)
System.out.println("no edge for " + parser.getVocabulary().getDisplayName(t));
int alt;
if (dfa_debug) {
Interval interval = Interval.of(startIndex, parser.getInputStream().index());
System.out.println("ATN exec upon " + parser.getInputStream().getText(interval) + " at DFA state " + s.stateNumber);
}
SimulatorState initialState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
alt = execATN(dfa, input, startIndex, initialState);
if (dfa_debug) {
System.out.println("back from DFA update, alt=" + alt + ", dfa=\n" + dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
// dump(dfa);
}
// action already executed
if (dfa_debug)
System.out.println("DFA decision " + dfa.decision + " predicts " + alt);
// we've updated DFA, exec'd action, and have our deepest answer
return alt;
} else if (target == ERROR) {
SimulatorState errorState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
return handleNoViableAlt(input, startIndex, errorState);
}
s = target;
if (!isAcceptState(s, state.useContext) && t != IntStream.EOF) {
input.consume();
t = input.LA(1);
}
}
if (!state.useContext && s.configs.getConflictInfo() != null) {
if (dfa.atnStartState instanceof DecisionState) {
if (!userWantsCtxSensitive || (!s.configs.getDipsIntoOuterContext() && s.configs.isExactConflict()) || (treat_sllk1_conflict_as_ambiguity && input.index() == startIndex)) {
// we don't report the ambiguity again
// if ( !acceptState.configset.hasSemanticContext() ) {
// reportAmbiguity(dfa, acceptState, startIndex, input.index(), acceptState.configset.getConflictingAlts(), acceptState.configset);
// }
} else {
assert !state.useContext;
// Before attempting full context prediction, check to see if there are
// disambiguating or validating predicates to evaluate which allow an
// immediate decision
BitSet conflictingAlts = null;
DFAState.PredPrediction[] predicates = s.predicates;
if (predicates != null) {
int conflictIndex = input.index();
if (conflictIndex != startIndex) {
input.seek(startIndex);
}
conflictingAlts = evalSemanticContext(predicates, outerContext, true);
if (conflictingAlts.cardinality() == 1) {
return conflictingAlts.nextSetBit(0);
}
if (conflictIndex != startIndex) {
// restore the index so reporting the fallback to full
// context occurs with the index at the correct spot
input.seek(conflictIndex);
}
}
if (reportAmbiguities) {
SimulatorState conflictState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
reportAttemptingFullContext(dfa, conflictingAlts, conflictState, startIndex, input.index());
}
input.seek(startIndex);
return adaptivePredict(input, dfa.decision, outerContext, true);
}
}
}
// Before jumping to prediction, check to see if there are
// disambiguating or validating predicates to evaluate
DFAState.PredPrediction[] predicates = s.predicates;
if (predicates != null) {
int stopIndex = input.index();
if (startIndex != stopIndex) {
input.seek(startIndex);
}
BitSet alts = evalSemanticContext(predicates, outerContext, reportAmbiguities && predictionMode == PredictionMode.LL_EXACT_AMBIG_DETECTION);
switch(alts.cardinality()) {
case 0:
throw noViableAlt(input, outerContext, s.configs, startIndex);
case 1:
return alts.nextSetBit(0);
default:
// set of ambig alts is reported.
if (startIndex != stopIndex) {
input.seek(stopIndex);
}
reportAmbiguity(dfa, s, startIndex, stopIndex, s.configs.isExactConflict(), alts, s.configs);
return alts.nextSetBit(0);
}
}
if (dfa_debug)
System.out.println("DFA decision " + dfa.decision + " predicts " + s.getPrediction());
return s.getPrediction();
}
use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method getTokenName.
@NotNull
public String getTokenName(int t) {
if (t == Token.EOF) {
return "EOF";
}
Vocabulary vocabulary = parser != null ? parser.getVocabulary() : VocabularyImpl.EMPTY_VOCABULARY;
String displayName = vocabulary.getDisplayName(t);
if (displayName.equals(Integer.toString(t))) {
return displayName;
}
return displayName + "<" + t + ">";
}
use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method execATN.
/**
* Performs ATN simulation to compute a predicted alternative based
* upon the remaining input, but also updates the DFA cache to avoid
* having to traverse the ATN again for the same input sequence.
*
* There are some key conditions we're looking for after computing a new
* set of ATN configs (proposed DFA state):
* if the set is empty, there is no viable alternative for current symbol
* does the state uniquely predict an alternative?
* does the state have a conflict that would prevent us from
* putting it on the work list?
* if in non-greedy decision is there a config at a rule stop state?
*
* We also have some key operations to do:
* add an edge from previous DFA state to potentially new DFA state, D,
* upon current symbol but only if adding to work list, which means in all
* cases except no viable alternative (and possibly non-greedy decisions?)
* collecting predicates and adding semantic context to DFA accept states
* adding rule context to context-sensitive DFA accept states
* consuming an input symbol
* reporting a conflict
* reporting an ambiguity
* reporting a context sensitivity
* reporting insufficient predicates
*
* We should isolate those operations, which are side-effecting, to the
* main work loop. We can isolate lots of code into other functions, but
* they should be side effect free. They can return package that
* indicates whether we should report something, whether we need to add a
* DFA edge, whether we need to augment accept state with semantic
* context or rule invocation context. Actually, it seems like we always
* add predicates if they exist, so that can simply be done in the main
* loop for any accept state creation or modification request.
*
* cover these cases:
* dead end
* single alt
* single alt + preds
* conflict
* conflict + preds
*
* TODO: greedy + those
*/
protected int execATN(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState initialState) {
if (debug)
System.out.println("execATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input));
final ParserRuleContext outerContext = initialState.outerContext;
final boolean useContext = initialState.useContext;
int t = input.LA(1);
SimulatorState previous = initialState;
PredictionContextCache contextCache = new PredictionContextCache();
while (true) {
// while more work
SimulatorState nextState = computeReachSet(dfa, previous, t, contextCache);
if (nextState == null) {
addDFAEdge(previous.s0, input.LA(1), ERROR);
return handleNoViableAlt(input, startIndex, previous);
}
DFAState D = nextState.s0;
// predicted alt => accept state
assert D.isAcceptState() || D.getPrediction() == ATN.INVALID_ALT_NUMBER;
// conflicted => accept state
assert D.isAcceptState() || D.configs.getConflictInfo() == null;
if (isAcceptState(D, useContext)) {
BitSet conflictingAlts = D.configs.getConflictingAlts();
int predictedAlt = conflictingAlts == null ? D.getPrediction() : ATN.INVALID_ALT_NUMBER;
if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
if (optimize_ll1 && input.index() == startIndex && !dfa.isPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.hasSemanticContext()) {
if (t >= 0 && t <= Short.MAX_VALUE) {
int key = (dfa.decision << 16) + t;
atn.LL1Table.put(key, predictedAlt);
}
}
if (useContext && always_try_local_context) {
reportContextSensitivity(dfa, predictedAlt, nextState, startIndex, input.index());
}
}
predictedAlt = D.getPrediction();
// int k = input.index() - startIndex + 1; // how much input we used
// System.out.println("used k="+k);
boolean attemptFullContext = conflictingAlts != null && userWantsCtxSensitive;
if (attemptFullContext) {
// Only exact conflicts are known to be ambiguous when local
// prediction does not step out of the decision rule.
attemptFullContext = !useContext && (D.configs.getDipsIntoOuterContext() || !D.configs.isExactConflict()) && (!treat_sllk1_conflict_as_ambiguity || input.index() != startIndex);
}
if (D.configs.hasSemanticContext()) {
DFAState.PredPrediction[] predPredictions = D.predicates;
if (predPredictions != null) {
int conflictIndex = input.index();
if (conflictIndex != startIndex) {
input.seek(startIndex);
}
// use complete evaluation here if we'll want to retry with full context if still ambiguous
conflictingAlts = evalSemanticContext(predPredictions, outerContext, attemptFullContext || reportAmbiguities);
switch(conflictingAlts.cardinality()) {
case 0:
throw noViableAlt(input, outerContext, D.configs, startIndex);
case 1:
return conflictingAlts.nextSetBit(0);
default:
break;
}
if (conflictIndex != startIndex) {
// restore the index so reporting the fallback to full
// context occurs with the index at the correct spot
input.seek(conflictIndex);
}
}
}
if (!attemptFullContext) {
if (conflictingAlts != null) {
if (reportAmbiguities && conflictingAlts.cardinality() > 1) {
reportAmbiguity(dfa, D, startIndex, input.index(), D.configs.isExactConflict(), conflictingAlts, D.configs);
}
predictedAlt = conflictingAlts.nextSetBit(0);
}
return predictedAlt;
} else {
assert !useContext;
assert isAcceptState(D, false);
if (debug)
System.out.println("RETRY with outerContext=" + outerContext);
SimulatorState fullContextState = computeStartState(dfa, outerContext, true);
if (reportAmbiguities) {
reportAttemptingFullContext(dfa, conflictingAlts, nextState, startIndex, input.index());
}
input.seek(startIndex);
return execATN(dfa, input, startIndex, fullContextState);
}
}
previous = nextState;
if (t != IntStream.EOF) {
input.consume();
t = input.LA(1);
}
}
}
use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.
the class ParserATNSimulator method computeStartState.
@NotNull
protected SimulatorState computeStartState(DFA dfa, ParserRuleContext globalContext, boolean useContext) {
DFAState s0 = dfa.isPrecedenceDfa() ? dfa.getPrecedenceStartState(parser.getPrecedence(), useContext) : useContext ? dfa.s0full.get() : dfa.s0.get();
if (s0 != null) {
if (!useContext) {
return new SimulatorState(globalContext, s0, useContext, globalContext);
}
s0.setContextSensitive(atn);
}
final int decision = dfa.decision;
@NotNull final ATNState p = dfa.atnStartState;
int previousContext = 0;
ParserRuleContext remainingGlobalContext = globalContext;
// always at least the implicit call to start rule
PredictionContext initialContext = useContext ? PredictionContext.EMPTY_FULL : PredictionContext.EMPTY_LOCAL;
PredictionContextCache contextCache = new PredictionContextCache();
if (useContext) {
if (!enable_global_context_dfa) {
while (remainingGlobalContext != null) {
if (remainingGlobalContext.isEmpty()) {
previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
remainingGlobalContext = null;
} else {
previousContext = getReturnState(remainingGlobalContext);
initialContext = initialContext.appendContext(previousContext, contextCache);
remainingGlobalContext = remainingGlobalContext.getParent();
}
}
}
while (s0 != null && s0.isContextSensitive() && remainingGlobalContext != null) {
DFAState next;
remainingGlobalContext = skipTailCalls(remainingGlobalContext);
if (remainingGlobalContext.isEmpty()) {
next = s0.getContextTarget(PredictionContext.EMPTY_FULL_STATE_KEY);
previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
remainingGlobalContext = null;
} else {
previousContext = getReturnState(remainingGlobalContext);
next = s0.getContextTarget(previousContext);
initialContext = initialContext.appendContext(previousContext, contextCache);
remainingGlobalContext = remainingGlobalContext.getParent();
}
if (next == null) {
break;
}
s0 = next;
}
}
if (s0 != null && !s0.isContextSensitive()) {
return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
}
ATNConfigSet configs = new ATNConfigSet();
while (true) {
ATNConfigSet reachIntermediate = new ATNConfigSet();
int n = p.getNumberOfTransitions();
for (int ti = 0; ti < n; ti++) {
// for each transition
ATNState target = p.transition(ti).target;
reachIntermediate.add(ATNConfig.create(target, ti + 1, initialContext));
}
boolean hasMoreContext = remainingGlobalContext != null;
if (!hasMoreContext) {
configs.setOutermostConfigSet(true);
}
final boolean collectPredicates = true;
closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false);
boolean stepIntoGlobal = configs.getDipsIntoOuterContext();
DFAState next;
if (useContext && !enable_global_context_dfa) {
s0 = addDFAState(dfa, configs, contextCache);
break;
} else if (s0 == null) {
if (!dfa.isPrecedenceDfa()) {
AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
next = addDFAState(dfa, configs, contextCache);
if (!reference.compareAndSet(null, next)) {
next = reference.get();
}
} else {
/* If this is a precedence DFA, we use applyPrecedenceFilter
* to convert the computed start state to a precedence start
* state. We then use DFA.setPrecedenceStartState to set the
* appropriate start state for the precedence level rather
* than simply setting DFA.s0.
*/
configs = applyPrecedenceFilter(configs, globalContext, contextCache);
next = addDFAState(dfa, configs, contextCache);
dfa.setPrecedenceStartState(parser.getPrecedence(), useContext, next);
}
} else {
if (dfa.isPrecedenceDfa()) {
configs = applyPrecedenceFilter(configs, globalContext, contextCache);
}
next = addDFAState(dfa, configs, contextCache);
s0.setContextTarget(previousContext, next);
}
s0 = next;
if (!useContext || !stepIntoGlobal) {
break;
}
// TODO: make sure it distinguishes empty stack states
next.setContextSensitive(atn);
configs.clear();
remainingGlobalContext = skipTailCalls(remainingGlobalContext);
int nextContextElement = getReturnState(remainingGlobalContext);
if (remainingGlobalContext.isEmpty()) {
remainingGlobalContext = null;
} else {
remainingGlobalContext = remainingGlobalContext.getParent();
}
if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
initialContext = initialContext.appendContext(nextContextElement, contextCache);
}
previousContext = nextContextElement;
}
return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
}
Aggregations