Search in sources :

Example 26 with NotNull

use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method getStartState.

protected SimulatorState getStartState(@NotNull DFA dfa, @NotNull TokenStream input, @NotNull ParserRuleContext outerContext, boolean useContext) {
    if (!useContext) {
        if (dfa.isPrecedenceDfa()) {
            // the start state for a precedence DFA depends on the current
            // parser precedence, and is provided by a DFA method.
            DFAState state = dfa.getPrecedenceStartState(parser.getPrecedence(), false);
            if (state == null) {
                return null;
            }
            return new SimulatorState(outerContext, state, false, outerContext);
        } else {
            if (dfa.s0.get() == null) {
                return null;
            }
            return new SimulatorState(outerContext, dfa.s0.get(), false, outerContext);
        }
    }
    if (!enable_global_context_dfa) {
        return null;
    }
    ParserRuleContext remainingContext = outerContext;
    assert outerContext != null;
    DFAState s0;
    if (dfa.isPrecedenceDfa()) {
        s0 = dfa.getPrecedenceStartState(parser.getPrecedence(), true);
    } else {
        s0 = dfa.s0full.get();
    }
    while (remainingContext != null && s0 != null && s0.isContextSensitive()) {
        remainingContext = skipTailCalls(remainingContext);
        s0 = s0.getContextTarget(getReturnState(remainingContext));
        if (remainingContext.isEmpty()) {
            assert s0 == null || !s0.isContextSensitive();
        } else {
            remainingContext = remainingContext.getParent();
        }
    }
    if (s0 == null) {
        return null;
    }
    return new SimulatorState(outerContext, s0, useContext, remainingContext);
}
Also used : ParserRuleContext(org.antlr.v4.runtime.ParserRuleContext) DFAState(org.antlr.v4.runtime.dfa.DFAState)

Example 27 with NotNull

use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execDFA.

protected int execDFA(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState state) {
    ParserRuleContext outerContext = state.outerContext;
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input) + ", outerContext=" + outerContext.toString(parser));
    if (dfa_debug)
        System.out.print(dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
    DFAState s = state.s0;
    int t = input.LA(1);
    ParserRuleContext remainingOuterContext = state.remainingOuterContext;
    while (true) {
        if (dfa_debug)
            System.out.println("DFA state " + s.stateNumber + " LA(1)==" + getLookaheadName(input));
        if (state.useContext) {
            while (s.isContextSymbol(t)) {
                DFAState next = null;
                if (remainingOuterContext != null) {
                    remainingOuterContext = skipTailCalls(remainingOuterContext);
                    next = s.getContextTarget(getReturnState(remainingOuterContext));
                }
                if (next == null) {
                    // fail over to ATN
                    SimulatorState initialState = new SimulatorState(state.outerContext, s, state.useContext, remainingOuterContext);
                    return execATN(dfa, input, startIndex, initialState);
                }
                assert remainingOuterContext != null;
                remainingOuterContext = remainingOuterContext.getParent();
                s = next;
            }
        }
        if (isAcceptState(s, state.useContext)) {
            if (s.predicates != null) {
                if (dfa_debug)
                    System.out.println("accept " + s);
            } else {
                if (dfa_debug)
                    System.out.println("accept; predict " + s.getPrediction() + " in state " + s.stateNumber);
            }
            // TODO: v3 dfa don't do this.
            break;
        }
        // t is not updated if one of these states is reached
        assert !isAcceptState(s, state.useContext);
        // if no edge, pop over to ATN interpreter, update DFA and return
        DFAState target = getExistingTargetState(s, t);
        if (target == null) {
            if (dfa_debug && t >= 0)
                System.out.println("no edge for " + parser.getVocabulary().getDisplayName(t));
            int alt;
            if (dfa_debug) {
                Interval interval = Interval.of(startIndex, parser.getInputStream().index());
                System.out.println("ATN exec upon " + parser.getInputStream().getText(interval) + " at DFA state " + s.stateNumber);
            }
            SimulatorState initialState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            alt = execATN(dfa, input, startIndex, initialState);
            if (dfa_debug) {
                System.out.println("back from DFA update, alt=" + alt + ", dfa=\n" + dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
            // dump(dfa);
            }
            // action already executed
            if (dfa_debug)
                System.out.println("DFA decision " + dfa.decision + " predicts " + alt);
            // we've updated DFA, exec'd action, and have our deepest answer
            return alt;
        } else if (target == ERROR) {
            SimulatorState errorState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            return handleNoViableAlt(input, startIndex, errorState);
        }
        s = target;
        if (!isAcceptState(s, state.useContext) && t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
    if (!state.useContext && s.configs.getConflictInfo() != null) {
        if (dfa.atnStartState instanceof DecisionState) {
            if (!userWantsCtxSensitive || (!s.configs.getDipsIntoOuterContext() && s.configs.isExactConflict()) || (treat_sllk1_conflict_as_ambiguity && input.index() == startIndex)) {
            // we don't report the ambiguity again
            // if ( !acceptState.configset.hasSemanticContext() ) {
            // reportAmbiguity(dfa, acceptState, startIndex, input.index(), acceptState.configset.getConflictingAlts(), acceptState.configset);
            // }
            } else {
                assert !state.useContext;
                // Before attempting full context prediction, check to see if there are
                // disambiguating or validating predicates to evaluate which allow an
                // immediate decision
                BitSet conflictingAlts = null;
                DFAState.PredPrediction[] predicates = s.predicates;
                if (predicates != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    conflictingAlts = evalSemanticContext(predicates, outerContext, true);
                    if (conflictingAlts.cardinality() == 1) {
                        return conflictingAlts.nextSetBit(0);
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
                if (reportAmbiguities) {
                    SimulatorState conflictState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
                    reportAttemptingFullContext(dfa, conflictingAlts, conflictState, startIndex, input.index());
                }
                input.seek(startIndex);
                return adaptivePredict(input, dfa.decision, outerContext, true);
            }
        }
    }
    // Before jumping to prediction, check to see if there are
    // disambiguating or validating predicates to evaluate
    DFAState.PredPrediction[] predicates = s.predicates;
    if (predicates != null) {
        int stopIndex = input.index();
        if (startIndex != stopIndex) {
            input.seek(startIndex);
        }
        BitSet alts = evalSemanticContext(predicates, outerContext, reportAmbiguities && predictionMode == PredictionMode.LL_EXACT_AMBIG_DETECTION);
        switch(alts.cardinality()) {
            case 0:
                throw noViableAlt(input, outerContext, s.configs, startIndex);
            case 1:
                return alts.nextSetBit(0);
            default:
                // set of ambig alts is reported.
                if (startIndex != stopIndex) {
                    input.seek(stopIndex);
                }
                reportAmbiguity(dfa, s, startIndex, stopIndex, s.configs.isExactConflict(), alts, s.configs);
                return alts.nextSetBit(0);
        }
    }
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " predicts " + s.getPrediction());
    return s.getPrediction();
}
Also used : ParserRuleContext(org.antlr.v4.runtime.ParserRuleContext) DFAState(org.antlr.v4.runtime.dfa.DFAState) BitSet(java.util.BitSet) Interval(org.antlr.v4.runtime.misc.Interval)

Example 28 with NotNull

use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method getTokenName.

@NotNull
public String getTokenName(int t) {
    if (t == Token.EOF) {
        return "EOF";
    }
    Vocabulary vocabulary = parser != null ? parser.getVocabulary() : VocabularyImpl.EMPTY_VOCABULARY;
    String displayName = vocabulary.getDisplayName(t);
    if (displayName.equals(Integer.toString(t))) {
        return displayName;
    }
    return displayName + "<" + t + ">";
}
Also used : Vocabulary(org.antlr.v4.runtime.Vocabulary) NotNull(org.antlr.v4.runtime.misc.NotNull)

Example 29 with NotNull

use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execATN.

/**
 * Performs ATN simulation to compute a predicted alternative based
 *  upon the remaining input, but also updates the DFA cache to avoid
 *  having to traverse the ATN again for the same input sequence.
 *
 *	 There are some key conditions we're looking for after computing a new
 *	 set of ATN configs (proposed DFA state):
 * if the set is empty, there is no viable alternative for current symbol
 * does the state uniquely predict an alternative?
 * does the state have a conflict that would prevent us from
 *	         putting it on the work list?
 * if in non-greedy decision is there a config at a rule stop state?
 *
 *	 We also have some key operations to do:
 * add an edge from previous DFA state to potentially new DFA state, D,
 *	         upon current symbol but only if adding to work list, which means in all
 *	         cases except no viable alternative (and possibly non-greedy decisions?)
 * collecting predicates and adding semantic context to DFA accept states
 * adding rule context to context-sensitive DFA accept states
 * consuming an input symbol
 * reporting a conflict
 * reporting an ambiguity
 * reporting a context sensitivity
 * reporting insufficient predicates
 *
 *	 We should isolate those operations, which are side-effecting, to the
 *	 main work loop. We can isolate lots of code into other functions, but
 *	 they should be side effect free. They can return package that
 *	 indicates whether we should report something, whether we need to add a
 *	 DFA edge, whether we need to augment accept state with semantic
 *	 context or rule invocation context. Actually, it seems like we always
 *	 add predicates if they exist, so that can simply be done in the main
 *	 loop for any accept state creation or modification request.
 *
 *	 cover these cases:
 *	    dead end
 *	    single alt
 *	    single alt + preds
 *	    conflict
 *	    conflict + preds
 *
 *	 TODO: greedy + those
 */
protected int execATN(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState initialState) {
    if (debug)
        System.out.println("execATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input));
    final ParserRuleContext outerContext = initialState.outerContext;
    final boolean useContext = initialState.useContext;
    int t = input.LA(1);
    SimulatorState previous = initialState;
    PredictionContextCache contextCache = new PredictionContextCache();
    while (true) {
        // while more work
        SimulatorState nextState = computeReachSet(dfa, previous, t, contextCache);
        if (nextState == null) {
            addDFAEdge(previous.s0, input.LA(1), ERROR);
            return handleNoViableAlt(input, startIndex, previous);
        }
        DFAState D = nextState.s0;
        // predicted alt => accept state
        assert D.isAcceptState() || D.getPrediction() == ATN.INVALID_ALT_NUMBER;
        // conflicted => accept state
        assert D.isAcceptState() || D.configs.getConflictInfo() == null;
        if (isAcceptState(D, useContext)) {
            BitSet conflictingAlts = D.configs.getConflictingAlts();
            int predictedAlt = conflictingAlts == null ? D.getPrediction() : ATN.INVALID_ALT_NUMBER;
            if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
                if (optimize_ll1 && input.index() == startIndex && !dfa.isPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.hasSemanticContext()) {
                    if (t >= 0 && t <= Short.MAX_VALUE) {
                        int key = (dfa.decision << 16) + t;
                        atn.LL1Table.put(key, predictedAlt);
                    }
                }
                if (useContext && always_try_local_context) {
                    reportContextSensitivity(dfa, predictedAlt, nextState, startIndex, input.index());
                }
            }
            predictedAlt = D.getPrediction();
            // int k = input.index() - startIndex + 1; // how much input we used
            // System.out.println("used k="+k);
            boolean attemptFullContext = conflictingAlts != null && userWantsCtxSensitive;
            if (attemptFullContext) {
                // Only exact conflicts are known to be ambiguous when local
                // prediction does not step out of the decision rule.
                attemptFullContext = !useContext && (D.configs.getDipsIntoOuterContext() || !D.configs.isExactConflict()) && (!treat_sllk1_conflict_as_ambiguity || input.index() != startIndex);
            }
            if (D.configs.hasSemanticContext()) {
                DFAState.PredPrediction[] predPredictions = D.predicates;
                if (predPredictions != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    // use complete evaluation here if we'll want to retry with full context if still ambiguous
                    conflictingAlts = evalSemanticContext(predPredictions, outerContext, attemptFullContext || reportAmbiguities);
                    switch(conflictingAlts.cardinality()) {
                        case 0:
                            throw noViableAlt(input, outerContext, D.configs, startIndex);
                        case 1:
                            return conflictingAlts.nextSetBit(0);
                        default:
                            break;
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
            }
            if (!attemptFullContext) {
                if (conflictingAlts != null) {
                    if (reportAmbiguities && conflictingAlts.cardinality() > 1) {
                        reportAmbiguity(dfa, D, startIndex, input.index(), D.configs.isExactConflict(), conflictingAlts, D.configs);
                    }
                    predictedAlt = conflictingAlts.nextSetBit(0);
                }
                return predictedAlt;
            } else {
                assert !useContext;
                assert isAcceptState(D, false);
                if (debug)
                    System.out.println("RETRY with outerContext=" + outerContext);
                SimulatorState fullContextState = computeStartState(dfa, outerContext, true);
                if (reportAmbiguities) {
                    reportAttemptingFullContext(dfa, conflictingAlts, nextState, startIndex, input.index());
                }
                input.seek(startIndex);
                return execATN(dfa, input, startIndex, fullContextState);
            }
        }
        previous = nextState;
        if (t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
}
Also used : ParserRuleContext(org.antlr.v4.runtime.ParserRuleContext) DFAState(org.antlr.v4.runtime.dfa.DFAState) BitSet(java.util.BitSet)

Example 30 with NotNull

use of org.antlr.v4.runtime.misc.NotNull in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method computeStartState.

@NotNull
protected SimulatorState computeStartState(DFA dfa, ParserRuleContext globalContext, boolean useContext) {
    DFAState s0 = dfa.isPrecedenceDfa() ? dfa.getPrecedenceStartState(parser.getPrecedence(), useContext) : useContext ? dfa.s0full.get() : dfa.s0.get();
    if (s0 != null) {
        if (!useContext) {
            return new SimulatorState(globalContext, s0, useContext, globalContext);
        }
        s0.setContextSensitive(atn);
    }
    final int decision = dfa.decision;
    @NotNull final ATNState p = dfa.atnStartState;
    int previousContext = 0;
    ParserRuleContext remainingGlobalContext = globalContext;
    // always at least the implicit call to start rule
    PredictionContext initialContext = useContext ? PredictionContext.EMPTY_FULL : PredictionContext.EMPTY_LOCAL;
    PredictionContextCache contextCache = new PredictionContextCache();
    if (useContext) {
        if (!enable_global_context_dfa) {
            while (remainingGlobalContext != null) {
                if (remainingGlobalContext.isEmpty()) {
                    previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                    remainingGlobalContext = null;
                } else {
                    previousContext = getReturnState(remainingGlobalContext);
                    initialContext = initialContext.appendContext(previousContext, contextCache);
                    remainingGlobalContext = remainingGlobalContext.getParent();
                }
            }
        }
        while (s0 != null && s0.isContextSensitive() && remainingGlobalContext != null) {
            DFAState next;
            remainingGlobalContext = skipTailCalls(remainingGlobalContext);
            if (remainingGlobalContext.isEmpty()) {
                next = s0.getContextTarget(PredictionContext.EMPTY_FULL_STATE_KEY);
                previousContext = PredictionContext.EMPTY_FULL_STATE_KEY;
                remainingGlobalContext = null;
            } else {
                previousContext = getReturnState(remainingGlobalContext);
                next = s0.getContextTarget(previousContext);
                initialContext = initialContext.appendContext(previousContext, contextCache);
                remainingGlobalContext = remainingGlobalContext.getParent();
            }
            if (next == null) {
                break;
            }
            s0 = next;
        }
    }
    if (s0 != null && !s0.isContextSensitive()) {
        return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
    }
    ATNConfigSet configs = new ATNConfigSet();
    while (true) {
        ATNConfigSet reachIntermediate = new ATNConfigSet();
        int n = p.getNumberOfTransitions();
        for (int ti = 0; ti < n; ti++) {
            // for each transition
            ATNState target = p.transition(ti).target;
            reachIntermediate.add(ATNConfig.create(target, ti + 1, initialContext));
        }
        boolean hasMoreContext = remainingGlobalContext != null;
        if (!hasMoreContext) {
            configs.setOutermostConfigSet(true);
        }
        final boolean collectPredicates = true;
        closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache, false);
        boolean stepIntoGlobal = configs.getDipsIntoOuterContext();
        DFAState next;
        if (useContext && !enable_global_context_dfa) {
            s0 = addDFAState(dfa, configs, contextCache);
            break;
        } else if (s0 == null) {
            if (!dfa.isPrecedenceDfa()) {
                AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
                next = addDFAState(dfa, configs, contextCache);
                if (!reference.compareAndSet(null, next)) {
                    next = reference.get();
                }
            } else {
                /* If this is a precedence DFA, we use applyPrecedenceFilter
					 * to convert the computed start state to a precedence start
					 * state. We then use DFA.setPrecedenceStartState to set the
					 * appropriate start state for the precedence level rather
					 * than simply setting DFA.s0.
					 */
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
                next = addDFAState(dfa, configs, contextCache);
                dfa.setPrecedenceStartState(parser.getPrecedence(), useContext, next);
            }
        } else {
            if (dfa.isPrecedenceDfa()) {
                configs = applyPrecedenceFilter(configs, globalContext, contextCache);
            }
            next = addDFAState(dfa, configs, contextCache);
            s0.setContextTarget(previousContext, next);
        }
        s0 = next;
        if (!useContext || !stepIntoGlobal) {
            break;
        }
        // TODO: make sure it distinguishes empty stack states
        next.setContextSensitive(atn);
        configs.clear();
        remainingGlobalContext = skipTailCalls(remainingGlobalContext);
        int nextContextElement = getReturnState(remainingGlobalContext);
        if (remainingGlobalContext.isEmpty()) {
            remainingGlobalContext = null;
        } else {
            remainingGlobalContext = remainingGlobalContext.getParent();
        }
        if (nextContextElement != PredictionContext.EMPTY_FULL_STATE_KEY) {
            initialContext = initialContext.appendContext(nextContextElement, contextCache);
        }
        previousContext = nextContextElement;
    }
    return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
}
Also used : ParserRuleContext(org.antlr.v4.runtime.ParserRuleContext) DFAState(org.antlr.v4.runtime.dfa.DFAState) AtomicReference(java.util.concurrent.atomic.AtomicReference) NotNull(org.antlr.v4.runtime.misc.NotNull) NotNull(org.antlr.v4.runtime.misc.NotNull)

Aggregations

NotNull (org.antlr.v4.runtime.misc.NotNull)40 ATNState (org.antlr.v4.runtime.atn.ATNState)14 IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)14 DFAState (org.antlr.v4.runtime.dfa.DFAState)12 ParseTree (org.antlr.v4.runtime.tree.ParseTree)9 ParserRuleContext (org.antlr.v4.runtime.ParserRuleContext)8 ArrayList (java.util.ArrayList)7 Token (org.antlr.v4.runtime.Token)7 RuleTransition (org.antlr.v4.runtime.atn.RuleTransition)4 DFA (org.antlr.v4.runtime.dfa.DFA)4 TerminalNode (org.antlr.v4.runtime.tree.TerminalNode)4 GrammarAST (org.antlr.v4.tool.ast.GrammarAST)4 QuantifierAST (org.antlr.v4.tool.ast.QuantifierAST)4 List (java.util.List)3 ATN (org.antlr.v4.runtime.atn.ATN)3 BlockEndState (org.antlr.v4.runtime.atn.BlockEndState)3 Transition (org.antlr.v4.runtime.atn.Transition)3 WildcardTransition (org.antlr.v4.runtime.atn.WildcardTransition)3 Interval (org.antlr.v4.runtime.misc.Interval)3 Nullable (org.antlr.v4.runtime.misc.Nullable)3