Examples with RuleStartState org.antlr.v4.runtime.atn.RuleStartState used on opensource projects

Example 11 with RuleStartState

use of org.antlr.v4.runtime.atn.RuleStartState in project antlr4 by antlr.

the class TestAmbigParseTrees method testInterpAtSpecificAlt.

void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g, String startRule, int startAlt, String input, String expectedParseTree) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
    Transition tr = ruleStartState.transition(0);
    ATNState t2 = tr.target;
    if (!(t2 instanceof BasicBlockStartState)) {
        throw new IllegalArgumentException("rule has no decision: " + startRule);
    }
    parser.addDecisionOverride(((DecisionState) t2).decision, 0, startAlt);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
}

Example 12 with RuleStartState

use of org.antlr.v4.runtime.atn.RuleStartState in project antlr4 by antlr.

the class ATNDeserializer method deserialize.

@SuppressWarnings("deprecation")
public ATN deserialize(char[] data) {
    data = data.clone();
    // was implemented.
    for (int i = 1; i < data.length; i++) {
        data[i] = (char) (data[i] - 2);
    }
    int p = 0;
    int version = toInt(data[p++]);
    if (version != SERIALIZED_VERSION) {
        String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with version %d (expected %d).", version, SERIALIZED_VERSION);
        throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
    }
    UUID uuid = toUUID(data, p);
    p += 8;
    if (!SUPPORTED_UUIDS.contains(uuid)) {
        String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with UUID %s (expected %s or a legacy UUID).", uuid, SERIALIZED_UUID);
        throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
    }
    boolean supportsPrecedencePredicates = isFeatureSupported(ADDED_PRECEDENCE_TRANSITIONS, uuid);
    boolean supportsLexerActions = isFeatureSupported(ADDED_LEXER_ACTIONS, uuid);
    ATNType grammarType = ATNType.values()[toInt(data[p++])];
    int maxTokenType = toInt(data[p++]);
    ATN atn = new ATN(grammarType, maxTokenType);
    //
    // STATES
    //
    List<Pair<LoopEndState, Integer>> loopBackStateNumbers = new ArrayList<Pair<LoopEndState, Integer>>();
    List<Pair<BlockStartState, Integer>> endStateNumbers = new ArrayList<Pair<BlockStartState, Integer>>();
    int nstates = toInt(data[p++]);
    for (int i = 0; i < nstates; i++) {
        int stype = toInt(data[p++]);
        // ignore bad type of states
        if (stype == ATNState.INVALID_TYPE) {
            atn.addState(null);
            continue;
        }
        int ruleIndex = toInt(data[p++]);
        if (ruleIndex == Character.MAX_VALUE) {
            ruleIndex = -1;
        }
        ATNState s = stateFactory(stype, ruleIndex);
        if (stype == ATNState.LOOP_END) {
            // special case
            int loopBackStateNumber = toInt(data[p++]);
            loopBackStateNumbers.add(new Pair<LoopEndState, Integer>((LoopEndState) s, loopBackStateNumber));
        } else if (s instanceof BlockStartState) {
            int endStateNumber = toInt(data[p++]);
            endStateNumbers.add(new Pair<BlockStartState, Integer>((BlockStartState) s, endStateNumber));
        }
        atn.addState(s);
    }
    // delay the assignment of loop back and end states until we know all the state instances have been initialized
    for (Pair<LoopEndState, Integer> pair : loopBackStateNumbers) {
        pair.a.loopBackState = atn.states.get(pair.b);
    }
    for (Pair<BlockStartState, Integer> pair : endStateNumbers) {
        pair.a.endState = (BlockEndState) atn.states.get(pair.b);
    }
    int numNonGreedyStates = toInt(data[p++]);
    for (int i = 0; i < numNonGreedyStates; i++) {
        int stateNumber = toInt(data[p++]);
        ((DecisionState) atn.states.get(stateNumber)).nonGreedy = true;
    }
    if (supportsPrecedencePredicates) {
        int numPrecedenceStates = toInt(data[p++]);
        for (int i = 0; i < numPrecedenceStates; i++) {
            int stateNumber = toInt(data[p++]);
            ((RuleStartState) atn.states.get(stateNumber)).isLeftRecursiveRule = true;
        }
    }
    //
    // RULES
    //
    int nrules = toInt(data[p++]);
    if (atn.grammarType == ATNType.LEXER) {
        atn.ruleToTokenType = new int[nrules];
    }
    atn.ruleToStartState = new RuleStartState[nrules];
    for (int i = 0; i < nrules; i++) {
        int s = toInt(data[p++]);
        RuleStartState startState = (RuleStartState) atn.states.get(s);
        atn.ruleToStartState[i] = startState;
        if (atn.grammarType == ATNType.LEXER) {
            int tokenType = toInt(data[p++]);
            if (tokenType == 0xFFFF) {
                tokenType = Token.EOF;
            }
            atn.ruleToTokenType[i] = tokenType;
            if (!isFeatureSupported(ADDED_LEXER_ACTIONS, uuid)) {
                // this piece of unused metadata was serialized prior to the
                // addition of LexerAction
                int actionIndexIgnored = toInt(data[p++]);
            }
        }
    }
    atn.ruleToStopState = new RuleStopState[nrules];
    for (ATNState state : atn.states) {
        if (!(state instanceof RuleStopState)) {
            continue;
        }
        RuleStopState stopState = (RuleStopState) state;
        atn.ruleToStopState[state.ruleIndex] = stopState;
        atn.ruleToStartState[state.ruleIndex].stopState = stopState;
    }
    //
    // MODES
    //
    int nmodes = toInt(data[p++]);
    for (int i = 0; i < nmodes; i++) {
        int s = toInt(data[p++]);
        atn.modeToStartState.add((TokensStartState) atn.states.get(s));
    }
    //
    // SETS
    //
    List<IntervalSet> sets = new ArrayList<IntervalSet>();
    // First, read all sets with 16-bit Unicode code points <= U+FFFF.
    p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP));
    // deserialize sets with 32-bit arguments <= U+10FFFF.
    if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) {
        p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP));
    }
    //
    // EDGES
    //
    int nedges = toInt(data[p++]);
    for (int i = 0; i < nedges; i++) {
        int src = toInt(data[p]);
        int trg = toInt(data[p + 1]);
        int ttype = toInt(data[p + 2]);
        int arg1 = toInt(data[p + 3]);
        int arg2 = toInt(data[p + 4]);
        int arg3 = toInt(data[p + 5]);
        Transition trans = edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets);
        //			System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+
        //							   src+"->"+trg+
        //					   " "+Transition.serializationNames[ttype]+
        //					   " "+arg1+","+arg2+","+arg3);
        ATNState srcState = atn.states.get(src);
        srcState.addTransition(trans);
        p += 6;
    }
    // edges for rule stop states can be derived, so they aren't serialized
    for (ATNState state : atn.states) {
        for (int i = 0; i < state.getNumberOfTransitions(); i++) {
            Transition t = state.transition(i);
            if (!(t instanceof RuleTransition)) {
                continue;
            }
            RuleTransition ruleTransition = (RuleTransition) t;
            int outermostPrecedenceReturn = -1;
            if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isLeftRecursiveRule) {
                if (ruleTransition.precedence == 0) {
                    outermostPrecedenceReturn = ruleTransition.target.ruleIndex;
                }
            }
            EpsilonTransition returnTransition = new EpsilonTransition(ruleTransition.followState, outermostPrecedenceReturn);
            atn.ruleToStopState[ruleTransition.target.ruleIndex].addTransition(returnTransition);
        }
    }
    for (ATNState state : atn.states) {
        if (state instanceof BlockStartState) {
            // we need to know the end state to set its start state
            if (((BlockStartState) state).endState == null) {
                throw new IllegalStateException();
            }
            // block end states can only be associated to a single block start state
            if (((BlockStartState) state).endState.startState != null) {
                throw new IllegalStateException();
            }
            ((BlockStartState) state).endState.startState = (BlockStartState) state;
        }
        if (state instanceof PlusLoopbackState) {
            PlusLoopbackState loopbackState = (PlusLoopbackState) state;
            for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
                ATNState target = loopbackState.transition(i).target;
                if (target instanceof PlusBlockStartState) {
                    ((PlusBlockStartState) target).loopBackState = loopbackState;
                }
            }
        } else if (state instanceof StarLoopbackState) {
            StarLoopbackState loopbackState = (StarLoopbackState) state;
            for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
                ATNState target = loopbackState.transition(i).target;
                if (target instanceof StarLoopEntryState) {
                    ((StarLoopEntryState) target).loopBackState = loopbackState;
                }
            }
        }
    }
    //
    // DECISIONS
    //
    int ndecisions = toInt(data[p++]);
    for (int i = 1; i <= ndecisions; i++) {
        int s = toInt(data[p++]);
        DecisionState decState = (DecisionState) atn.states.get(s);
        atn.decisionToState.add(decState);
        decState.decision = i - 1;
    }
    //
    if (atn.grammarType == ATNType.LEXER) {
        if (supportsLexerActions) {
            atn.lexerActions = new LexerAction[toInt(data[p++])];
            for (int i = 0; i < atn.lexerActions.length; i++) {
                LexerActionType actionType = LexerActionType.values()[toInt(data[p++])];
                int data1 = toInt(data[p++]);
                if (data1 == 0xFFFF) {
                    data1 = -1;
                }
                int data2 = toInt(data[p++]);
                if (data2 == 0xFFFF) {
                    data2 = -1;
                }
                LexerAction lexerAction = lexerActionFactory(actionType, data1, data2);
                atn.lexerActions[i] = lexerAction;
            }
        } else {
            // for compatibility with older serialized ATNs, convert the old
            // serialized action index for action transitions to the new
            // form, which is the index of a LexerCustomAction
            List<LexerAction> legacyLexerActions = new ArrayList<LexerAction>();
            for (ATNState state : atn.states) {
                for (int i = 0; i < state.getNumberOfTransitions(); i++) {
                    Transition transition = state.transition(i);
                    if (!(transition instanceof ActionTransition)) {
                        continue;
                    }
                    int ruleIndex = ((ActionTransition) transition).ruleIndex;
                    int actionIndex = ((ActionTransition) transition).actionIndex;
                    LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex);
                    state.setTransition(i, new ActionTransition(transition.target, ruleIndex, legacyLexerActions.size(), false));
                    legacyLexerActions.add(lexerAction);
                }
            }
            atn.lexerActions = legacyLexerActions.toArray(new LexerAction[legacyLexerActions.size()]);
        }
    }
    markPrecedenceDecisions(atn);
    if (deserializationOptions.isVerifyATN()) {
        verifyATN(atn);
    }
    if (deserializationOptions.isGenerateRuleBypassTransitions() && atn.grammarType == ATNType.PARSER) {
        atn.ruleToTokenType = new int[atn.ruleToStartState.length];
        for (int i = 0; i < atn.ruleToStartState.length; i++) {
            atn.ruleToTokenType[i] = atn.maxTokenType + i + 1;
        }
        for (int i = 0; i < atn.ruleToStartState.length; i++) {
            BasicBlockStartState bypassStart = new BasicBlockStartState();
            bypassStart.ruleIndex = i;
            atn.addState(bypassStart);
            BlockEndState bypassStop = new BlockEndState();
            bypassStop.ruleIndex = i;
            atn.addState(bypassStop);
            bypassStart.endState = bypassStop;
            atn.defineDecisionState(bypassStart);
            bypassStop.startState = bypassStart;
            ATNState endState;
            Transition excludeTransition = null;
            if (atn.ruleToStartState[i].isLeftRecursiveRule) {
                // wrap from the beginning of the rule to the StarLoopEntryState
                endState = null;
                for (ATNState state : atn.states) {
                    if (state.ruleIndex != i) {
                        continue;
                    }
                    if (!(state instanceof StarLoopEntryState)) {
                        continue;
                    }
                    ATNState maybeLoopEndState = state.transition(state.getNumberOfTransitions() - 1).target;
                    if (!(maybeLoopEndState instanceof LoopEndState)) {
                        continue;
                    }
                    if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
                        endState = state;
                        break;
                    }
                }
                if (endState == null) {
                    throw new UnsupportedOperationException("Couldn't identify final state of the precedence rule prefix section.");
                }
                excludeTransition = ((StarLoopEntryState) endState).loopBackState.transition(0);
            } else {
                endState = atn.ruleToStopState[i];
            }
            // all non-excluded transitions that currently target end state need to target blockEnd instead
            for (ATNState state : atn.states) {
                for (Transition transition : state.transitions) {
                    if (transition == excludeTransition) {
                        continue;
                    }
                    if (transition.target == endState) {
                        transition.target = bypassStop;
                    }
                }
            }
            // all transitions leaving the rule start state need to leave blockStart instead
            while (atn.ruleToStartState[i].getNumberOfTransitions() > 0) {
                Transition transition = atn.ruleToStartState[i].removeTransition(atn.ruleToStartState[i].getNumberOfTransitions() - 1);
                bypassStart.addTransition(transition);
            }
            // link the new states
            atn.ruleToStartState[i].addTransition(new EpsilonTransition(bypassStart));
            bypassStop.addTransition(new EpsilonTransition(endState));
            ATNState matchState = new BasicState();
            atn.addState(matchState);
            matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i]));
            bypassStart.addTransition(new EpsilonTransition(matchState));
        }
        if (deserializationOptions.isVerifyATN()) {
            // reverify after modification
            verifyATN(atn);
        }
    }
    return atn;
}

Example 13 with RuleStartState

use of org.antlr.v4.runtime.atn.RuleStartState in project antlr4 by antlr.

the class ParserInterpreter method parse.

/** Begin parsing at startRuleIndex */
public ParserRuleContext parse(int startRuleIndex) {
    RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex];
    rootContext = createInterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
    if (startRuleStartState.isLeftRecursiveRule) {
        enterRecursionRule(rootContext, startRuleStartState.stateNumber, startRuleIndex, 0);
    } else {
        enterRule(rootContext, startRuleStartState.stateNumber, startRuleIndex);
    }
    while (true) {
        ATNState p = getATNState();
        switch(p.getStateType()) {
            case ATNState.RULE_STOP:
                // pop; return from rule
                if (_ctx.isEmpty()) {
                    if (startRuleStartState.isLeftRecursiveRule) {
                        ParserRuleContext result = _ctx;
                        Pair<ParserRuleContext, Integer> parentContext = _parentContextStack.pop();
                        unrollRecursionContexts(parentContext.a);
                        return result;
                    } else {
                        exitRule();
                        return rootContext;
                    }
                }
                visitRuleStopState(p);
                break;
            default:
                try {
                    visitState(p);
                } catch (RecognitionException e) {
                    setState(atn.ruleToStopState[p.ruleIndex].stateNumber);
                    getContext().exception = e;
                    getErrorHandler().reportError(this, e);
                    recover(e);
                }
                break;
        }
    }
}

Example 14 with RuleStartState

use of org.antlr.v4.runtime.atn.RuleStartState in project antlr4 by antlr.

the class LexerATNFactory method createATN.

@Override
public ATN createATN() {
    // BUILD ALL START STATES (ONE PER MODE)
    Set<String> modes = ((LexerGrammar) g).modes.keySet();
    for (String modeName : modes) {
        // create s0, start state; implied Tokens rule node
        TokensStartState startState = newState(TokensStartState.class, null);
        atn.modeNameToStartState.put(modeName, startState);
        atn.modeToStartState.add(startState);
        atn.defineDecisionState(startState);
    }
    // INIT ACTION, RULE->TOKEN_TYPE MAP
    atn.ruleToTokenType = new int[g.rules.size()];
    for (Rule r : g.rules.values()) {
        atn.ruleToTokenType[r.index] = g.getTokenType(r.name);
    }
    // CREATE ATN FOR EACH RULE
    _createATN(g.rules.values());
    atn.lexerActions = new LexerAction[indexToActionMap.size()];
    for (Map.Entry<Integer, LexerAction> entry : indexToActionMap.entrySet()) {
        atn.lexerActions[entry.getKey()] = entry.getValue();
    }
    // LINK MODE START STATE TO EACH TOKEN RULE
    for (String modeName : modes) {
        List<Rule> rules = ((LexerGrammar) g).modes.get(modeName);
        TokensStartState startState = atn.modeNameToStartState.get(modeName);
        for (Rule r : rules) {
            if (!r.isFragment()) {
                RuleStartState s = atn.ruleToStartState[r.index];
                epsilon(startState, s);
            }
        }
    }
    ATNOptimizer.optimize(g, atn);
    return atn;
}

Example 15 with RuleStartState

use of org.antlr.v4.runtime.atn.RuleStartState in project antlr4 by antlr.

the class GrammarParserInterpreter method findOuterMostDecisionStates.

/** identify the ATN states where we need to set the outer alt number.
	 *  For regular rules, that's the block at the target to rule start state.
	 *  For left-recursive rules, we track the primary block, which looks just
	 *  like a regular rule's outer block, and the star loop block (always
	 *  there even if 1 alt).
	 */
public BitSet findOuterMostDecisionStates() {
    BitSet track = new BitSet(atn.states.size());
    int numberOfDecisions = atn.getNumberOfDecisions();
    for (int i = 0; i < numberOfDecisions; i++) {
        DecisionState decisionState = atn.getDecisionState(i);
        RuleStartState startState = atn.ruleToStartState[decisionState.ruleIndex];
        // Look for StarLoopEntryState that is in any left recursive rule
        if (decisionState instanceof StarLoopEntryState) {
            StarLoopEntryState loopEntry = (StarLoopEntryState) decisionState;
            if (loopEntry.isPrecedenceDecision) {
                // Recursive alts always result in a (...)* in the transformed
                // left recursive rule and that always has a BasicBlockStartState
                // even if just 1 recursive alt exists.
                ATNState blockStart = loopEntry.transition(0).target;
                // track the StarBlockStartState associated with the recursive alternatives
                track.set(blockStart.stateNumber);
            }
        } else if (startState.transition(0).target == decisionState) {
            // always track outermost block for any rule if it exists
            track.set(decisionState.stateNumber);
        }
    }
    return track;
}