Search in sources :

Example 1 with DecisionState

use of org.antlr.v4.runtime.atn.DecisionState in project antlr4 by antlr.

the class ATNSerializer method serialize.

/** Serialize state descriptors, edge descriptors, and decision→state map
	 *  into list of ints:
	 *
	 * 		grammar-type, (ANTLRParser.LEXER, ...)
	 *  	max token type,
	 *  	num states,
	 *  	state-0-type ruleIndex, state-1-type ruleIndex, ... state-i-type ruleIndex optional-arg ...
	 *  	num rules,
	 *  	rule-1-start-state rule-1-args, rule-2-start-state  rule-2-args, ...
	 *  	(args are token type,actionIndex in lexer else 0,0)
	 *      num modes,
	 *      mode-0-start-state, mode-1-start-state, ... (parser has 0 modes)
	 *      num unicode-bmp-sets
	 *      bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ...
	 *      num unicode-smp-sets
	 *      smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ...
	 *	num total edges,
	 *      src, trg, edge-type, edge arg1, optional edge arg2 (present always), ...
	 *      num decisions,
	 *      decision-0-start-state, decision-1-start-state, ...
	 *
	 *  Convenient to pack into unsigned shorts to make as Java string.
	 */
public IntegerList serialize() {
    IntegerList data = new IntegerList();
    data.add(ATNDeserializer.SERIALIZED_VERSION);
    serializeUUID(data, ATNDeserializer.SERIALIZED_UUID);
    // convert grammar type to ATN const to avoid dependence on ANTLRParser
    data.add(atn.grammarType.ordinal());
    data.add(atn.maxTokenType);
    int nedges = 0;
    // Note that we use a LinkedHashMap as a set to
    // maintain insertion order while deduplicating
    // entries with the same key.
    Map<IntervalSet, Boolean> sets = new LinkedHashMap<>();
    // dump states, count edges and collect sets while doing so
    IntegerList nonGreedyStates = new IntegerList();
    IntegerList precedenceStates = new IntegerList();
    data.add(atn.states.size());
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            data.add(ATNState.INVALID_TYPE);
            continue;
        }
        int stateType = s.getStateType();
        if (s instanceof DecisionState && ((DecisionState) s).nonGreedy) {
            nonGreedyStates.add(s.stateNumber);
        }
        if (s instanceof RuleStartState && ((RuleStartState) s).isLeftRecursiveRule) {
            precedenceStates.add(s.stateNumber);
        }
        data.add(stateType);
        if (s.ruleIndex == -1) {
            data.add(Character.MAX_VALUE);
        } else {
            data.add(s.ruleIndex);
        }
        if (s.getStateType() == ATNState.LOOP_END) {
            data.add(((LoopEndState) s).loopBackState.stateNumber);
        } else if (s instanceof BlockStartState) {
            data.add(((BlockStartState) s).endState.stateNumber);
        }
        if (s.getStateType() != ATNState.RULE_STOP) {
            // the deserializer can trivially derive these edges, so there's no need to serialize them
            nedges += s.getNumberOfTransitions();
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            int edgeType = Transition.serializationTypes.get(t.getClass());
            if (edgeType == Transition.SET || edgeType == Transition.NOT_SET) {
                SetTransition st = (SetTransition) t;
                sets.put(st.set, true);
            }
        }
    }
    // non-greedy states
    data.add(nonGreedyStates.size());
    for (int i = 0; i < nonGreedyStates.size(); i++) {
        data.add(nonGreedyStates.get(i));
    }
    // precedence states
    data.add(precedenceStates.size());
    for (int i = 0; i < precedenceStates.size(); i++) {
        data.add(precedenceStates.get(i));
    }
    int nrules = atn.ruleToStartState.length;
    data.add(nrules);
    for (int r = 0; r < nrules; r++) {
        ATNState ruleStartState = atn.ruleToStartState[r];
        data.add(ruleStartState.stateNumber);
        if (atn.grammarType == ATNType.LEXER) {
            if (atn.ruleToTokenType[r] == Token.EOF) {
                data.add(Character.MAX_VALUE);
            } else {
                data.add(atn.ruleToTokenType[r]);
            }
        }
    }
    int nmodes = atn.modeToStartState.size();
    data.add(nmodes);
    if (nmodes > 0) {
        for (ATNState modeStartState : atn.modeToStartState) {
            data.add(modeStartState.stateNumber);
        }
    }
    List<IntervalSet> bmpSets = new ArrayList<>();
    List<IntervalSet> smpSets = new ArrayList<>();
    for (IntervalSet set : sets.keySet()) {
        if (set.getMaxElement() <= Character.MAX_VALUE) {
            bmpSets.add(set);
        } else {
            smpSets.add(set);
        }
    }
    serializeSets(data, bmpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            data.add(cp);
        }
    });
    serializeSets(data, smpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            serializeInt(data, cp);
        }
    });
    Map<IntervalSet, Integer> setIndices = new HashMap<>();
    int setIndex = 0;
    for (IntervalSet bmpSet : bmpSets) {
        setIndices.put(bmpSet, setIndex++);
    }
    for (IntervalSet smpSet : smpSets) {
        setIndices.put(smpSet, setIndex++);
    }
    data.add(nedges);
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            continue;
        }
        if (s.getStateType() == ATNState.RULE_STOP) {
            continue;
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            if (atn.states.get(t.target.stateNumber) == null) {
                throw new IllegalStateException("Cannot serialize a transition to a removed state.");
            }
            int src = s.stateNumber;
            int trg = t.target.stateNumber;
            int edgeType = Transition.serializationTypes.get(t.getClass());
            int arg1 = 0;
            int arg2 = 0;
            int arg3 = 0;
            switch(edgeType) {
                case Transition.RULE:
                    trg = ((RuleTransition) t).followState.stateNumber;
                    arg1 = ((RuleTransition) t).target.stateNumber;
                    arg2 = ((RuleTransition) t).ruleIndex;
                    arg3 = ((RuleTransition) t).precedence;
                    break;
                case Transition.PRECEDENCE:
                    PrecedencePredicateTransition ppt = (PrecedencePredicateTransition) t;
                    arg1 = ppt.precedence;
                    break;
                case Transition.PREDICATE:
                    PredicateTransition pt = (PredicateTransition) t;
                    arg1 = pt.ruleIndex;
                    arg2 = pt.predIndex;
                    arg3 = pt.isCtxDependent ? 1 : 0;
                    break;
                case Transition.RANGE:
                    arg1 = ((RangeTransition) t).from;
                    arg2 = ((RangeTransition) t).to;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ATOM:
                    arg1 = ((AtomTransition) t).label;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ACTION:
                    ActionTransition at = (ActionTransition) t;
                    arg1 = at.ruleIndex;
                    arg2 = at.actionIndex;
                    if (arg2 == -1) {
                        arg2 = 0xFFFF;
                    }
                    arg3 = at.isCtxDependent ? 1 : 0;
                    break;
                case Transition.SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.NOT_SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.WILDCARD:
                    break;
            }
            data.add(src);
            data.add(trg);
            data.add(edgeType);
            data.add(arg1);
            data.add(arg2);
            data.add(arg3);
        }
    }
    int ndecisions = atn.decisionToState.size();
    data.add(ndecisions);
    for (DecisionState decStartState : atn.decisionToState) {
        data.add(decStartState.stateNumber);
    }
    //
    if (atn.grammarType == ATNType.LEXER) {
        data.add(atn.lexerActions.length);
        for (LexerAction action : atn.lexerActions) {
            data.add(action.getActionType().ordinal());
            switch(action.getActionType()) {
                case CHANNEL:
                    int channel = ((LexerChannelAction) action).getChannel();
                    data.add(channel != -1 ? channel : 0xFFFF);
                    data.add(0);
                    break;
                case CUSTOM:
                    int ruleIndex = ((LexerCustomAction) action).getRuleIndex();
                    int actionIndex = ((LexerCustomAction) action).getActionIndex();
                    data.add(ruleIndex != -1 ? ruleIndex : 0xFFFF);
                    data.add(actionIndex != -1 ? actionIndex : 0xFFFF);
                    break;
                case MODE:
                    int mode = ((LexerModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case MORE:
                    data.add(0);
                    data.add(0);
                    break;
                case POP_MODE:
                    data.add(0);
                    data.add(0);
                    break;
                case PUSH_MODE:
                    mode = ((LexerPushModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case SKIP:
                    data.add(0);
                    data.add(0);
                    break;
                case TYPE:
                    int type = ((LexerTypeAction) action).getType();
                    data.add(type != -1 ? type : 0xFFFF);
                    data.add(0);
                    break;
                default:
                    String message = String.format(Locale.getDefault(), "The specified lexer action type %s is not valid.", action.getActionType());
                    throw new IllegalArgumentException(message);
            }
        }
    }
    // don't adjust the first value since that's the version number
    for (int i = 1; i < data.size(); i++) {
        if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) {
            throw new UnsupportedOperationException("Serialized ATN data element " + data.get(i) + " element " + i + " out of range " + (int) Character.MIN_VALUE + ".." + (int) Character.MAX_VALUE);
        }
        int value = (data.get(i) + 2) & 0xFFFF;
        data.set(i, value);
    }
    return data;
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) IntegerList(org.antlr.v4.runtime.misc.IntegerList) IntervalSet(org.antlr.v4.runtime.misc.IntervalSet)

Example 2 with DecisionState

use of org.antlr.v4.runtime.atn.DecisionState in project antlr4 by antlr.

the class ParserInterpreter method visitState.

protected void visitState(ATNState p) {
    //		System.out.println("visitState "+p.stateNumber);
    int predictedAlt = 1;
    if (p instanceof DecisionState) {
        predictedAlt = visitDecisionState((DecisionState) p);
    }
    Transition transition = p.transition(predictedAlt - 1);
    switch(transition.getSerializationType()) {
        case Transition.EPSILON:
            if (p.getStateType() == ATNState.STAR_LOOP_ENTRY && ((StarLoopEntryState) p).isPrecedenceDecision && !(transition.target instanceof LoopEndState)) {
                // We are at the start of a left recursive rule's (...)* loop
                // and we're not taking the exit branch of loop.
                InterpreterRuleContext localctx = createInterpreterRuleContext(_parentContextStack.peek().a, _parentContextStack.peek().b, _ctx.getRuleIndex());
                pushNewRecursionContext(localctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.getRuleIndex());
            }
            break;
        case Transition.ATOM:
            match(((AtomTransition) transition).label);
            break;
        case Transition.RANGE:
        case Transition.SET:
        case Transition.NOT_SET:
            if (!transition.matches(_input.LA(1), Token.MIN_USER_TOKEN_TYPE, 65535)) {
                recoverInline();
            }
            matchWildcard();
            break;
        case Transition.WILDCARD:
            matchWildcard();
            break;
        case Transition.RULE:
            RuleStartState ruleStartState = (RuleStartState) transition.target;
            int ruleIndex = ruleStartState.ruleIndex;
            InterpreterRuleContext newctx = createInterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
            if (ruleStartState.isLeftRecursiveRule) {
                enterRecursionRule(newctx, ruleStartState.stateNumber, ruleIndex, ((RuleTransition) transition).precedence);
            } else {
                enterRule(newctx, transition.target.stateNumber, ruleIndex);
            }
            break;
        case Transition.PREDICATE:
            PredicateTransition predicateTransition = (PredicateTransition) transition;
            if (!sempred(_ctx, predicateTransition.ruleIndex, predicateTransition.predIndex)) {
                throw new FailedPredicateException(this);
            }
            break;
        case Transition.ACTION:
            ActionTransition actionTransition = (ActionTransition) transition;
            action(_ctx, actionTransition.ruleIndex, actionTransition.actionIndex);
            break;
        case Transition.PRECEDENCE:
            if (!precpred(_ctx, ((PrecedencePredicateTransition) transition).precedence)) {
                throw new FailedPredicateException(this, String.format("precpred(_ctx, %d)", ((PrecedencePredicateTransition) transition).precedence));
            }
            break;
        default:
            throw new UnsupportedOperationException("Unrecognized ATN transition type.");
    }
    setState(transition.target.stateNumber);
}
Also used : LoopEndState(org.antlr.v4.runtime.atn.LoopEndState) ActionTransition(org.antlr.v4.runtime.atn.ActionTransition) StarLoopEntryState(org.antlr.v4.runtime.atn.StarLoopEntryState) RuleStartState(org.antlr.v4.runtime.atn.RuleStartState) AtomTransition(org.antlr.v4.runtime.atn.AtomTransition) RuleTransition(org.antlr.v4.runtime.atn.RuleTransition) ActionTransition(org.antlr.v4.runtime.atn.ActionTransition) PredicateTransition(org.antlr.v4.runtime.atn.PredicateTransition) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) Transition(org.antlr.v4.runtime.atn.Transition) PredicateTransition(org.antlr.v4.runtime.atn.PredicateTransition) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) DecisionState(org.antlr.v4.runtime.atn.DecisionState)

Example 3 with DecisionState

use of org.antlr.v4.runtime.atn.DecisionState in project antlr4 by antlr.

the class GrammarParserInterpreter method getLookaheadParseTrees.

/** Return a list of parse trees, one for each alternative in a decision
	 *  given the same input.
	 *
	 *  Very similar to {@link #getAllPossibleParseTrees} except
	 *  that it re-parses the input for every alternative in a decision,
	 *  not just the ambiguous ones (there is no alts parameter here).
	 *  This method also tries to reduce the size of the parse trees
	 *  by stripping away children of the tree that are completely out of range
	 *  of startIndex..stopIndex. Also, because errors are expected, we
	 *  use a specialized error handler that more or less bails out
	 *  but that also consumes the first erroneous token at least. This
	 *  ensures that an error node will be in the parse tree for display.
	 *
	 *  NOTES:
    // we must parse the entire input now with decision overrides
	// we cannot parse a subset because it could be that a decision
	// above our decision of interest needs to read way past
	// lookaheadInfo.stopIndex. It seems like there is no escaping
	// the use of a full and complete token stream if we are
	// resetting to token index 0 and re-parsing from the start symbol.
	// It's not easy to restart parsing somewhere in the middle like a
	// continuation because our call stack does not match the
	// tree stack because of left recursive rule rewriting. grrrr!
	 *
	 * @since 4.5.1
	 */
public static List<ParserRuleContext> getLookaheadParseTrees(Grammar g, ParserInterpreter originalParser, TokenStream tokens, int startRuleIndex, int decision, int startIndex, int stopIndex) {
    List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>();
    // Create a new parser interpreter to parse the ambiguous subphrase
    ParserInterpreter parser = deriveTempParserInterpreter(g, originalParser, tokens);
    DecisionState decisionState = originalParser.getATN().decisionToState.get(decision);
    for (int alt = 1; alt <= decisionState.getTransitions().length; alt++) {
        // re-parse entire input for all ambiguous alternatives
        // (don't have to do first as it's been parsed, but do again for simplicity
        //  using this temp parser.)
        GrammarParserInterpreter.BailButConsumeErrorStrategy errorHandler = new GrammarParserInterpreter.BailButConsumeErrorStrategy();
        parser.setErrorHandler(errorHandler);
        parser.reset();
        parser.addDecisionOverride(decision, startIndex, alt);
        ParserRuleContext tt = parser.parse(startRuleIndex);
        int stopTreeAt = stopIndex;
        if (errorHandler.firstErrorTokenIndex >= 0) {
            // cut off rest at first error
            stopTreeAt = errorHandler.firstErrorTokenIndex;
        }
        Interval overallRange = tt.getSourceInterval();
        if (stopTreeAt > overallRange.b) {
            // If we try to look beyond range of tree, stopTreeAt must be EOF
            // for which there is no EOF ref in grammar. That means tree
            // will not have node for stopTreeAt; limit to overallRange.b
            stopTreeAt = overallRange.b;
        }
        ParserRuleContext subtree = Trees.getRootOfSubtreeEnclosingRegion(tt, startIndex, stopTreeAt);
        // Use higher of overridden decision tree or tree enclosing all tokens
        if (Trees.isAncestorOf(parser.getOverrideDecisionRoot(), subtree)) {
            subtree = parser.getOverrideDecisionRoot();
        }
        Trees.stripChildrenOutOfRange(subtree, parser.getOverrideDecisionRoot(), startIndex, stopTreeAt);
        trees.add(subtree);
    }
    return trees;
}
Also used : ParserRuleContext(org.antlr.v4.runtime.ParserRuleContext) ParserInterpreter(org.antlr.v4.runtime.ParserInterpreter) ArrayList(java.util.ArrayList) DecisionState(org.antlr.v4.runtime.atn.DecisionState) Interval(org.antlr.v4.runtime.misc.Interval)

Example 4 with DecisionState

use of org.antlr.v4.runtime.atn.DecisionState in project antlr4 by antlr.

the class BaseCppTest method checkRuleDFA.

void checkRuleDFA(Grammar g, DecisionState blk, String expecting) throws Exception {
    DFA dfa = createDFA(g, blk);
    String result = null;
    if (dfa != null)
        result = dfa.toString();
    assertEquals(expecting, result);
}
Also used : STGroupString(org.stringtemplate.v4.STGroupString) BaseRuntimeTest.antlrOnString(org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString) DFA(org.antlr.v4.runtime.dfa.DFA)

Example 5 with DecisionState

use of org.antlr.v4.runtime.atn.DecisionState in project antlr4 by antlr.

the class BaseJavaTest method checkRuleDFA.

List<ANTLRMessage> checkRuleDFA(String gtext, String ruleName, String expecting) throws Exception {
    ErrorQueue equeue = new ErrorQueue();
    Grammar g = new Grammar(gtext, equeue);
    ATN atn = createATN(g, false);
    ATNState s = atn.ruleToStartState[g.getRule(ruleName).index];
    if (s == null) {
        System.err.println("no such rule: " + ruleName);
        return null;
    }
    ATNState t = s.transition(0).target;
    if (!(t instanceof DecisionState)) {
        System.out.println(ruleName + " has no decision");
        return null;
    }
    DecisionState blk = (DecisionState) t;
    checkRuleDFA(g, blk, expecting);
    return equeue.all;
}
Also used : ErrorQueue(org.antlr.v4.test.runtime.ErrorQueue) Grammar(org.antlr.v4.tool.Grammar) LexerGrammar(org.antlr.v4.tool.LexerGrammar) ATN(org.antlr.v4.runtime.atn.ATN) DecisionState(org.antlr.v4.runtime.atn.DecisionState) ATNState(org.antlr.v4.runtime.atn.ATNState)

Aggregations

DecisionState (org.antlr.v4.runtime.atn.DecisionState)13 ATNState (org.antlr.v4.runtime.atn.ATNState)8 ATN (org.antlr.v4.runtime.atn.ATN)6 ErrorQueue (org.antlr.v4.test.runtime.ErrorQueue)6 Grammar (org.antlr.v4.tool.Grammar)6 LexerGrammar (org.antlr.v4.tool.LexerGrammar)6 RuleStartState (org.antlr.v4.runtime.atn.RuleStartState)5 Transition (org.antlr.v4.runtime.atn.Transition)5 IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)5 ArrayList (java.util.ArrayList)4 AtomTransition (org.antlr.v4.runtime.atn.AtomTransition)4 ActionTransition (org.antlr.v4.runtime.atn.ActionTransition)3 RangeTransition (org.antlr.v4.runtime.atn.RangeTransition)3 RuleTransition (org.antlr.v4.runtime.atn.RuleTransition)3 SetTransition (org.antlr.v4.runtime.atn.SetTransition)3 StarLoopEntryState (org.antlr.v4.runtime.atn.StarLoopEntryState)3 DFA (org.antlr.v4.runtime.dfa.DFA)3 Interval (org.antlr.v4.runtime.misc.Interval)3 STGroupString (org.stringtemplate.v4.STGroupString)3 HashMap (java.util.HashMap)2