Examples with DFAState org.antlr.v4.runtime.dfa.DFAState used on opensource projects

Example 1 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class DOTGenerator method getDOT.

public String getDOT(DFA dfa, boolean isLexer) {
    if (dfa.s0.get() == null)
        return null;
    ST dot = stlib.getInstanceOf("dfa");
    dot.add("name", "DFA" + dfa.decision);
    dot.add("startState", dfa.s0.get().stateNumber);
    // dot.add("useBox", Tool.internalOption_ShowATNConfigsInDFA);
    dot.add("rankdir", rankdir);
    // define stop states first; seems to be a bug in DOT where doublecircle
    for (DFAState d : dfa.states.keySet()) {
        if (!d.isAcceptState())
            continue;
        ST st = stlib.getInstanceOf("stopstate");
        st.add("name", "s" + d.stateNumber);
        st.add("label", getStateLabel(d));
        dot.add("states", st);
    }
    for (DFAState d : dfa.states.keySet()) {
        if (d.isAcceptState())
            continue;
        if (d.stateNumber == Integer.MAX_VALUE)
            continue;
        ST st = stlib.getInstanceOf("state");
        st.add("name", "s" + d.stateNumber);
        st.add("label", getStateLabel(d));
        dot.add("states", st);
    }
    for (DFAState d : dfa.states.keySet()) {
        Map<Integer, DFAState> edges = d.getEdgeMap();
        for (Map.Entry<Integer, DFAState> entry : edges.entrySet()) {
            DFAState target = entry.getValue();
            if (target == null)
                continue;
            if (target.stateNumber == Integer.MAX_VALUE)
                continue;
            int ttype = entry.getKey();
            String label = String.valueOf(ttype);
            if (isLexer)
                label = "'" + getEdgeLabel(new StringBuilder().appendCodePoint(entry.getKey()).toString()) + "'";
            else if (grammar != null)
                label = grammar.getTokenDisplayName(ttype);
            ST st = stlib.getInstanceOf("edge");
            st.add("label", label);
            st.add("src", "s" + d.stateNumber);
            st.add("target", "s" + target.stateNumber);
            st.add("arrowhead", arrowhead);
            dot.add("edges", st);
        }
    }
    String output = dot.render();
    return Utils.sortLinesInString(output);
}

Example 2 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class DOTGenerator method getStateLabel.

protected String getStateLabel(DFAState s) {
    if (s == null)
        return "null";
    StringBuilder buf = new StringBuilder(250);
    buf.append('s');
    buf.append(s.stateNumber);
    if (s.isAcceptState()) {
        buf.append("=>").append(s.getPrediction());
    }
    if (grammar != null) {
        BitSet alts = s.configs.getRepresentedAlternatives();
        buf.append("\\n");
        Set<ATNConfig> configurations = s.configs;
        for (int alt = alts.nextSetBit(0); alt >= 0; alt = alts.nextSetBit(alt + 1)) {
            if (alt > alts.nextSetBit(0)) {
                buf.append("\\n");
            }
            buf.append("alt");
            buf.append(alt);
            buf.append(':');
            // get a list of configs for just this alt
            // it will help us print better later
            List<ATNConfig> configsInAlt = new ArrayList<ATNConfig>();
            for (ATNConfig c : configurations) {
                if (c.getAlt() != alt)
                    continue;
                configsInAlt.add(c);
            }
            int n = 0;
            for (int cIndex = 0; cIndex < configsInAlt.size(); cIndex++) {
                ATNConfig c = configsInAlt.get(cIndex);
                n++;
                buf.append(c.toString(null, false));
                if ((cIndex + 1) < configsInAlt.size()) {
                    buf.append(", ");
                }
                if (n % 5 == 0 && (configsInAlt.size() - cIndex) > 3) {
                    buf.append("\\n");
                }
            }
        }
    }
    String stateLabel = buf.toString();
    return stateLabel;
}

Example 3 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method getStartState.

protected SimulatorState getStartState(@NotNull DFA dfa, @NotNull TokenStream input, @NotNull ParserRuleContext outerContext, boolean useContext) {
    if (!useContext) {
        if (dfa.isPrecedenceDfa()) {
            // the start state for a precedence DFA depends on the current
            // parser precedence, and is provided by a DFA method.
            DFAState state = dfa.getPrecedenceStartState(parser.getPrecedence(), false);
            if (state == null) {
                return null;
            }
            return new SimulatorState(outerContext, state, false, outerContext);
        } else {
            if (dfa.s0.get() == null) {
                return null;
            }
            return new SimulatorState(outerContext, dfa.s0.get(), false, outerContext);
        }
    }
    if (!enable_global_context_dfa) {
        return null;
    }
    ParserRuleContext remainingContext = outerContext;
    assert outerContext != null;
    DFAState s0;
    if (dfa.isPrecedenceDfa()) {
        s0 = dfa.getPrecedenceStartState(parser.getPrecedence(), true);
    } else {
        s0 = dfa.s0full.get();
    }
    while (remainingContext != null && s0 != null && s0.isContextSensitive()) {
        remainingContext = skipTailCalls(remainingContext);
        s0 = s0.getContextTarget(getReturnState(remainingContext));
        if (remainingContext.isEmpty()) {
            assert s0 == null || !s0.isContextSensitive();
        } else {
            remainingContext = remainingContext.getParent();
        }
    }
    if (s0 == null) {
        return null;
    }
    return new SimulatorState(outerContext, s0, useContext, remainingContext);
}

Example 4 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execDFA.

protected int execDFA(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState state) {
    ParserRuleContext outerContext = state.outerContext;
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input) + ", outerContext=" + outerContext.toString(parser));
    if (dfa_debug)
        System.out.print(dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
    DFAState s = state.s0;
    int t = input.LA(1);
    ParserRuleContext remainingOuterContext = state.remainingOuterContext;
    while (true) {
        if (dfa_debug)
            System.out.println("DFA state " + s.stateNumber + " LA(1)==" + getLookaheadName(input));
        if (state.useContext) {
            while (s.isContextSymbol(t)) {
                DFAState next = null;
                if (remainingOuterContext != null) {
                    remainingOuterContext = skipTailCalls(remainingOuterContext);
                    next = s.getContextTarget(getReturnState(remainingOuterContext));
                }
                if (next == null) {
                    // fail over to ATN
                    SimulatorState initialState = new SimulatorState(state.outerContext, s, state.useContext, remainingOuterContext);
                    return execATN(dfa, input, startIndex, initialState);
                }
                assert remainingOuterContext != null;
                remainingOuterContext = remainingOuterContext.getParent();
                s = next;
            }
        }
        if (isAcceptState(s, state.useContext)) {
            if (s.predicates != null) {
                if (dfa_debug)
                    System.out.println("accept " + s);
            } else {
                if (dfa_debug)
                    System.out.println("accept; predict " + s.getPrediction() + " in state " + s.stateNumber);
            }
            // TODO: v3 dfa don't do this.
            break;
        }
        // t is not updated if one of these states is reached
        assert !isAcceptState(s, state.useContext);
        // if no edge, pop over to ATN interpreter, update DFA and return
        DFAState target = getExistingTargetState(s, t);
        if (target == null) {
            if (dfa_debug && t >= 0)
                System.out.println("no edge for " + parser.getVocabulary().getDisplayName(t));
            int alt;
            if (dfa_debug) {
                Interval interval = Interval.of(startIndex, parser.getInputStream().index());
                System.out.println("ATN exec upon " + parser.getInputStream().getText(interval) + " at DFA state " + s.stateNumber);
            }
            SimulatorState initialState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            alt = execATN(dfa, input, startIndex, initialState);
            if (dfa_debug) {
                System.out.println("back from DFA update, alt=" + alt + ", dfa=\n" + dfa.toString(parser.getVocabulary(), parser.getRuleNames()));
            // dump(dfa);
            }
            // action already executed
            if (dfa_debug)
                System.out.println("DFA decision " + dfa.decision + " predicts " + alt);
            // we've updated DFA, exec'd action, and have our deepest answer
            return alt;
        } else if (target == ERROR) {
            SimulatorState errorState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
            return handleNoViableAlt(input, startIndex, errorState);
        }
        s = target;
        if (!isAcceptState(s, state.useContext) && t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
    if (!state.useContext && s.configs.getConflictInfo() != null) {
        if (dfa.atnStartState instanceof DecisionState) {
            if (!userWantsCtxSensitive || (!s.configs.getDipsIntoOuterContext() && s.configs.isExactConflict()) || (treat_sllk1_conflict_as_ambiguity && input.index() == startIndex)) {
            // we don't report the ambiguity again
            // if ( !acceptState.configset.hasSemanticContext() ) {
            // reportAmbiguity(dfa, acceptState, startIndex, input.index(), acceptState.configset.getConflictingAlts(), acceptState.configset);
            // }
            } else {
                assert !state.useContext;
                // Before attempting full context prediction, check to see if there are
                // disambiguating or validating predicates to evaluate which allow an
                // immediate decision
                BitSet conflictingAlts = null;
                DFAState.PredPrediction[] predicates = s.predicates;
                if (predicates != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    conflictingAlts = evalSemanticContext(predicates, outerContext, true);
                    if (conflictingAlts.cardinality() == 1) {
                        return conflictingAlts.nextSetBit(0);
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
                if (reportAmbiguities) {
                    SimulatorState conflictState = new SimulatorState(outerContext, s, state.useContext, remainingOuterContext);
                    reportAttemptingFullContext(dfa, conflictingAlts, conflictState, startIndex, input.index());
                }
                input.seek(startIndex);
                return adaptivePredict(input, dfa.decision, outerContext, true);
            }
        }
    }
    // Before jumping to prediction, check to see if there are
    // disambiguating or validating predicates to evaluate
    DFAState.PredPrediction[] predicates = s.predicates;
    if (predicates != null) {
        int stopIndex = input.index();
        if (startIndex != stopIndex) {
            input.seek(startIndex);
        }
        BitSet alts = evalSemanticContext(predicates, outerContext, reportAmbiguities && predictionMode == PredictionMode.LL_EXACT_AMBIG_DETECTION);
        switch(alts.cardinality()) {
            case 0:
                throw noViableAlt(input, outerContext, s.configs, startIndex);
            case 1:
                return alts.nextSetBit(0);
            default:
                // set of ambig alts is reported.
                if (startIndex != stopIndex) {
                    input.seek(stopIndex);
                }
                reportAmbiguity(dfa, s, startIndex, stopIndex, s.configs.isExactConflict(), alts, s.configs);
                return alts.nextSetBit(0);
        }
    }
    if (dfa_debug)
        System.out.println("DFA decision " + dfa.decision + " predicts " + s.getPrediction());
    return s.getPrediction();
}

Example 5 with DFAState

use of org.antlr.v4.runtime.dfa.DFAState in project antlr4 by tunnelvisionlabs.

the class ParserATNSimulator method execATN.

/**
 * Performs ATN simulation to compute a predicted alternative based
 *  upon the remaining input, but also updates the DFA cache to avoid
 *  having to traverse the ATN again for the same input sequence.
 *
 *	 There are some key conditions we're looking for after computing a new
 *	 set of ATN configs (proposed DFA state):
 * if the set is empty, there is no viable alternative for current symbol
 * does the state uniquely predict an alternative?
 * does the state have a conflict that would prevent us from
 *	         putting it on the work list?
 * if in non-greedy decision is there a config at a rule stop state?
 *
 *	 We also have some key operations to do:
 * add an edge from previous DFA state to potentially new DFA state, D,
 *	         upon current symbol but only if adding to work list, which means in all
 *	         cases except no viable alternative (and possibly non-greedy decisions?)
 * collecting predicates and adding semantic context to DFA accept states
 * adding rule context to context-sensitive DFA accept states
 * consuming an input symbol
 * reporting a conflict
 * reporting an ambiguity
 * reporting a context sensitivity
 * reporting insufficient predicates
 *
 *	 We should isolate those operations, which are side-effecting, to the
 *	 main work loop. We can isolate lots of code into other functions, but
 *	 they should be side effect free. They can return package that
 *	 indicates whether we should report something, whether we need to add a
 *	 DFA edge, whether we need to augment accept state with semantic
 *	 context or rule invocation context. Actually, it seems like we always
 *	 add predicates if they exist, so that can simply be done in the main
 *	 loop for any accept state creation or modification request.
 *
 *	 cover these cases:
 *	    dead end
 *	    single alt
 *	    single alt + preds
 *	    conflict
 *	    conflict + preds
 *
 *	 TODO: greedy + those
 */
protected int execATN(@NotNull DFA dfa, @NotNull TokenStream input, int startIndex, @NotNull SimulatorState initialState) {
    if (debug)
        System.out.println("execATN decision " + dfa.decision + " exec LA(1)==" + getLookaheadName(input));
    final ParserRuleContext outerContext = initialState.outerContext;
    final boolean useContext = initialState.useContext;
    int t = input.LA(1);
    SimulatorState previous = initialState;
    PredictionContextCache contextCache = new PredictionContextCache();
    while (true) {
        // while more work
        SimulatorState nextState = computeReachSet(dfa, previous, t, contextCache);
        if (nextState == null) {
            addDFAEdge(previous.s0, input.LA(1), ERROR);
            return handleNoViableAlt(input, startIndex, previous);
        }
        DFAState D = nextState.s0;
        // predicted alt => accept state
        assert D.isAcceptState() || D.getPrediction() == ATN.INVALID_ALT_NUMBER;
        // conflicted => accept state
        assert D.isAcceptState() || D.configs.getConflictInfo() == null;
        if (isAcceptState(D, useContext)) {
            BitSet conflictingAlts = D.configs.getConflictingAlts();
            int predictedAlt = conflictingAlts == null ? D.getPrediction() : ATN.INVALID_ALT_NUMBER;
            if (predictedAlt != ATN.INVALID_ALT_NUMBER) {
                if (optimize_ll1 && input.index() == startIndex && !dfa.isPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.hasSemanticContext()) {
                    if (t >= 0 && t <= Short.MAX_VALUE) {
                        int key = (dfa.decision << 16) + t;
                        atn.LL1Table.put(key, predictedAlt);
                    }
                }
                if (useContext && always_try_local_context) {
                    reportContextSensitivity(dfa, predictedAlt, nextState, startIndex, input.index());
                }
            }
            predictedAlt = D.getPrediction();
            // int k = input.index() - startIndex + 1; // how much input we used
            // System.out.println("used k="+k);
            boolean attemptFullContext = conflictingAlts != null && userWantsCtxSensitive;
            if (attemptFullContext) {
                // Only exact conflicts are known to be ambiguous when local
                // prediction does not step out of the decision rule.
                attemptFullContext = !useContext && (D.configs.getDipsIntoOuterContext() || !D.configs.isExactConflict()) && (!treat_sllk1_conflict_as_ambiguity || input.index() != startIndex);
            }
            if (D.configs.hasSemanticContext()) {
                DFAState.PredPrediction[] predPredictions = D.predicates;
                if (predPredictions != null) {
                    int conflictIndex = input.index();
                    if (conflictIndex != startIndex) {
                        input.seek(startIndex);
                    }
                    // use complete evaluation here if we'll want to retry with full context if still ambiguous
                    conflictingAlts = evalSemanticContext(predPredictions, outerContext, attemptFullContext || reportAmbiguities);
                    switch(conflictingAlts.cardinality()) {
                        case 0:
                            throw noViableAlt(input, outerContext, D.configs, startIndex);
                        case 1:
                            return conflictingAlts.nextSetBit(0);
                        default:
                            break;
                    }
                    if (conflictIndex != startIndex) {
                        // restore the index so reporting the fallback to full
                        // context occurs with the index at the correct spot
                        input.seek(conflictIndex);
                    }
                }
            }
            if (!attemptFullContext) {
                if (conflictingAlts != null) {
                    if (reportAmbiguities && conflictingAlts.cardinality() > 1) {
                        reportAmbiguity(dfa, D, startIndex, input.index(), D.configs.isExactConflict(), conflictingAlts, D.configs);
                    }
                    predictedAlt = conflictingAlts.nextSetBit(0);
                }
                return predictedAlt;
            } else {
                assert !useContext;
                assert isAcceptState(D, false);
                if (debug)
                    System.out.println("RETRY with outerContext=" + outerContext);
                SimulatorState fullContextState = computeStartState(dfa, outerContext, true);
                if (reportAmbiguities) {
                    reportAttemptingFullContext(dfa, conflictingAlts, nextState, startIndex, input.index());
                }
                input.seek(startIndex);
                return execATN(dfa, input, startIndex, fullContextState);
            }
        }
        previous = nextState;
        if (t != IntStream.EOF) {
            input.consume();
            t = input.LA(1);
        }
    }
}