Search in sources :

Example 11 with LexerAction

use of org.antlr.v4.runtime.atn.LexerAction in project antlr4 by tunnelvisionlabs.

the class LexerATNFactory method action.

@Override
public Handle action(ActionAST action) {
    int ruleIndex = currentRule.index;
    int actionIndex = g.lexerActions.get(action);
    LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex);
    return action(action, lexerAction);
}
Also used : LexerCustomAction(org.antlr.v4.runtime.atn.LexerCustomAction)

Example 12 with LexerAction

use of org.antlr.v4.runtime.atn.LexerAction in project antlr4 by tunnelvisionlabs.

the class LexerATNFactory method action.

protected Handle action(GrammarAST node, LexerAction lexerAction) {
    ATNState left = newState(node);
    ATNState right = newState(node);
    boolean isCtxDependent = false;
    int lexerActionIndex = getLexerActionIndex(lexerAction);
    ActionTransition a = new ActionTransition(right, currentRule.index, lexerActionIndex, isCtxDependent);
    left.addTransition(a);
    node.atnState = left;
    Handle h = new Handle(left, right);
    return h;
}
Also used : ActionTransition(org.antlr.v4.runtime.atn.ActionTransition) ATNState(org.antlr.v4.runtime.atn.ATNState)

Example 13 with LexerAction

use of org.antlr.v4.runtime.atn.LexerAction in project antlr4 by antlr.

the class ATNDeserializer method deserialize.

@SuppressWarnings("deprecation")
public ATN deserialize(char[] data) {
    data = data.clone();
    // was implemented.
    for (int i = 1; i < data.length; i++) {
        data[i] = (char) (data[i] - 2);
    }
    int p = 0;
    int version = toInt(data[p++]);
    if (version != SERIALIZED_VERSION) {
        String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with version %d (expected %d).", version, SERIALIZED_VERSION);
        throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
    }
    UUID uuid = toUUID(data, p);
    p += 8;
    if (!SUPPORTED_UUIDS.contains(uuid)) {
        String reason = String.format(Locale.getDefault(), "Could not deserialize ATN with UUID %s (expected %s or a legacy UUID).", uuid, SERIALIZED_UUID);
        throw new UnsupportedOperationException(new InvalidClassException(ATN.class.getName(), reason));
    }
    boolean supportsPrecedencePredicates = isFeatureSupported(ADDED_PRECEDENCE_TRANSITIONS, uuid);
    boolean supportsLexerActions = isFeatureSupported(ADDED_LEXER_ACTIONS, uuid);
    ATNType grammarType = ATNType.values()[toInt(data[p++])];
    int maxTokenType = toInt(data[p++]);
    ATN atn = new ATN(grammarType, maxTokenType);
    // 
    // STATES
    // 
    List<Pair<LoopEndState, Integer>> loopBackStateNumbers = new ArrayList<Pair<LoopEndState, Integer>>();
    List<Pair<BlockStartState, Integer>> endStateNumbers = new ArrayList<Pair<BlockStartState, Integer>>();
    int nstates = toInt(data[p++]);
    for (int i = 0; i < nstates; i++) {
        int stype = toInt(data[p++]);
        // ignore bad type of states
        if (stype == ATNState.INVALID_TYPE) {
            atn.addState(null);
            continue;
        }
        int ruleIndex = toInt(data[p++]);
        if (ruleIndex == Character.MAX_VALUE) {
            ruleIndex = -1;
        }
        ATNState s = stateFactory(stype, ruleIndex);
        if (stype == ATNState.LOOP_END) {
            // special case
            int loopBackStateNumber = toInt(data[p++]);
            loopBackStateNumbers.add(new Pair<LoopEndState, Integer>((LoopEndState) s, loopBackStateNumber));
        } else if (s instanceof BlockStartState) {
            int endStateNumber = toInt(data[p++]);
            endStateNumbers.add(new Pair<BlockStartState, Integer>((BlockStartState) s, endStateNumber));
        }
        atn.addState(s);
    }
    // delay the assignment of loop back and end states until we know all the state instances have been initialized
    for (Pair<LoopEndState, Integer> pair : loopBackStateNumbers) {
        pair.a.loopBackState = atn.states.get(pair.b);
    }
    for (Pair<BlockStartState, Integer> pair : endStateNumbers) {
        pair.a.endState = (BlockEndState) atn.states.get(pair.b);
    }
    int numNonGreedyStates = toInt(data[p++]);
    for (int i = 0; i < numNonGreedyStates; i++) {
        int stateNumber = toInt(data[p++]);
        ((DecisionState) atn.states.get(stateNumber)).nonGreedy = true;
    }
    if (supportsPrecedencePredicates) {
        int numPrecedenceStates = toInt(data[p++]);
        for (int i = 0; i < numPrecedenceStates; i++) {
            int stateNumber = toInt(data[p++]);
            ((RuleStartState) atn.states.get(stateNumber)).isLeftRecursiveRule = true;
        }
    }
    // 
    // RULES
    // 
    int nrules = toInt(data[p++]);
    if (atn.grammarType == ATNType.LEXER) {
        atn.ruleToTokenType = new int[nrules];
    }
    atn.ruleToStartState = new RuleStartState[nrules];
    for (int i = 0; i < nrules; i++) {
        int s = toInt(data[p++]);
        RuleStartState startState = (RuleStartState) atn.states.get(s);
        atn.ruleToStartState[i] = startState;
        if (atn.grammarType == ATNType.LEXER) {
            int tokenType = toInt(data[p++]);
            if (tokenType == 0xFFFF) {
                tokenType = Token.EOF;
            }
            atn.ruleToTokenType[i] = tokenType;
            if (!isFeatureSupported(ADDED_LEXER_ACTIONS, uuid)) {
                // this piece of unused metadata was serialized prior to the
                // addition of LexerAction
                int actionIndexIgnored = toInt(data[p++]);
            }
        }
    }
    atn.ruleToStopState = new RuleStopState[nrules];
    for (ATNState state : atn.states) {
        if (!(state instanceof RuleStopState)) {
            continue;
        }
        RuleStopState stopState = (RuleStopState) state;
        atn.ruleToStopState[state.ruleIndex] = stopState;
        atn.ruleToStartState[state.ruleIndex].stopState = stopState;
    }
    // 
    // MODES
    // 
    int nmodes = toInt(data[p++]);
    for (int i = 0; i < nmodes; i++) {
        int s = toInt(data[p++]);
        atn.modeToStartState.add((TokensStartState) atn.states.get(s));
    }
    // 
    // SETS
    // 
    List<IntervalSet> sets = new ArrayList<IntervalSet>();
    // First, read all sets with 16-bit Unicode code points <= U+FFFF.
    p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP));
    // deserialize sets with 32-bit arguments <= U+10FFFF.
    if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) {
        p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP));
    }
    // 
    // EDGES
    // 
    int nedges = toInt(data[p++]);
    for (int i = 0; i < nedges; i++) {
        int src = toInt(data[p]);
        int trg = toInt(data[p + 1]);
        int ttype = toInt(data[p + 2]);
        int arg1 = toInt(data[p + 3]);
        int arg2 = toInt(data[p + 4]);
        int arg3 = toInt(data[p + 5]);
        Transition trans = edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets);
        // System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+
        // src+"->"+trg+
        // " "+Transition.serializationNames[ttype]+
        // " "+arg1+","+arg2+","+arg3);
        ATNState srcState = atn.states.get(src);
        srcState.addTransition(trans);
        p += 6;
    }
    // edges for rule stop states can be derived, so they aren't serialized
    for (ATNState state : atn.states) {
        for (int i = 0; i < state.getNumberOfTransitions(); i++) {
            Transition t = state.transition(i);
            if (!(t instanceof RuleTransition)) {
                continue;
            }
            RuleTransition ruleTransition = (RuleTransition) t;
            int outermostPrecedenceReturn = -1;
            if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isLeftRecursiveRule) {
                if (ruleTransition.precedence == 0) {
                    outermostPrecedenceReturn = ruleTransition.target.ruleIndex;
                }
            }
            EpsilonTransition returnTransition = new EpsilonTransition(ruleTransition.followState, outermostPrecedenceReturn);
            atn.ruleToStopState[ruleTransition.target.ruleIndex].addTransition(returnTransition);
        }
    }
    for (ATNState state : atn.states) {
        if (state instanceof BlockStartState) {
            // we need to know the end state to set its start state
            if (((BlockStartState) state).endState == null) {
                throw new IllegalStateException();
            }
            // block end states can only be associated to a single block start state
            if (((BlockStartState) state).endState.startState != null) {
                throw new IllegalStateException();
            }
            ((BlockStartState) state).endState.startState = (BlockStartState) state;
        }
        if (state instanceof PlusLoopbackState) {
            PlusLoopbackState loopbackState = (PlusLoopbackState) state;
            for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
                ATNState target = loopbackState.transition(i).target;
                if (target instanceof PlusBlockStartState) {
                    ((PlusBlockStartState) target).loopBackState = loopbackState;
                }
            }
        } else if (state instanceof StarLoopbackState) {
            StarLoopbackState loopbackState = (StarLoopbackState) state;
            for (int i = 0; i < loopbackState.getNumberOfTransitions(); i++) {
                ATNState target = loopbackState.transition(i).target;
                if (target instanceof StarLoopEntryState) {
                    ((StarLoopEntryState) target).loopBackState = loopbackState;
                }
            }
        }
    }
    // 
    // DECISIONS
    // 
    int ndecisions = toInt(data[p++]);
    for (int i = 1; i <= ndecisions; i++) {
        int s = toInt(data[p++]);
        DecisionState decState = (DecisionState) atn.states.get(s);
        atn.decisionToState.add(decState);
        decState.decision = i - 1;
    }
    // 
    if (atn.grammarType == ATNType.LEXER) {
        if (supportsLexerActions) {
            atn.lexerActions = new LexerAction[toInt(data[p++])];
            for (int i = 0; i < atn.lexerActions.length; i++) {
                LexerActionType actionType = LexerActionType.values()[toInt(data[p++])];
                int data1 = toInt(data[p++]);
                if (data1 == 0xFFFF) {
                    data1 = -1;
                }
                int data2 = toInt(data[p++]);
                if (data2 == 0xFFFF) {
                    data2 = -1;
                }
                LexerAction lexerAction = lexerActionFactory(actionType, data1, data2);
                atn.lexerActions[i] = lexerAction;
            }
        } else {
            // for compatibility with older serialized ATNs, convert the old
            // serialized action index for action transitions to the new
            // form, which is the index of a LexerCustomAction
            List<LexerAction> legacyLexerActions = new ArrayList<LexerAction>();
            for (ATNState state : atn.states) {
                for (int i = 0; i < state.getNumberOfTransitions(); i++) {
                    Transition transition = state.transition(i);
                    if (!(transition instanceof ActionTransition)) {
                        continue;
                    }
                    int ruleIndex = ((ActionTransition) transition).ruleIndex;
                    int actionIndex = ((ActionTransition) transition).actionIndex;
                    LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex);
                    state.setTransition(i, new ActionTransition(transition.target, ruleIndex, legacyLexerActions.size(), false));
                    legacyLexerActions.add(lexerAction);
                }
            }
            atn.lexerActions = legacyLexerActions.toArray(new LexerAction[legacyLexerActions.size()]);
        }
    }
    markPrecedenceDecisions(atn);
    if (deserializationOptions.isVerifyATN()) {
        verifyATN(atn);
    }
    if (deserializationOptions.isGenerateRuleBypassTransitions() && atn.grammarType == ATNType.PARSER) {
        atn.ruleToTokenType = new int[atn.ruleToStartState.length];
        for (int i = 0; i < atn.ruleToStartState.length; i++) {
            atn.ruleToTokenType[i] = atn.maxTokenType + i + 1;
        }
        for (int i = 0; i < atn.ruleToStartState.length; i++) {
            BasicBlockStartState bypassStart = new BasicBlockStartState();
            bypassStart.ruleIndex = i;
            atn.addState(bypassStart);
            BlockEndState bypassStop = new BlockEndState();
            bypassStop.ruleIndex = i;
            atn.addState(bypassStop);
            bypassStart.endState = bypassStop;
            atn.defineDecisionState(bypassStart);
            bypassStop.startState = bypassStart;
            ATNState endState;
            Transition excludeTransition = null;
            if (atn.ruleToStartState[i].isLeftRecursiveRule) {
                // wrap from the beginning of the rule to the StarLoopEntryState
                endState = null;
                for (ATNState state : atn.states) {
                    if (state.ruleIndex != i) {
                        continue;
                    }
                    if (!(state instanceof StarLoopEntryState)) {
                        continue;
                    }
                    ATNState maybeLoopEndState = state.transition(state.getNumberOfTransitions() - 1).target;
                    if (!(maybeLoopEndState instanceof LoopEndState)) {
                        continue;
                    }
                    if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.transition(0).target instanceof RuleStopState) {
                        endState = state;
                        break;
                    }
                }
                if (endState == null) {
                    throw new UnsupportedOperationException("Couldn't identify final state of the precedence rule prefix section.");
                }
                excludeTransition = ((StarLoopEntryState) endState).loopBackState.transition(0);
            } else {
                endState = atn.ruleToStopState[i];
            }
            // all non-excluded transitions that currently target end state need to target blockEnd instead
            for (ATNState state : atn.states) {
                for (Transition transition : state.transitions) {
                    if (transition == excludeTransition) {
                        continue;
                    }
                    if (transition.target == endState) {
                        transition.target = bypassStop;
                    }
                }
            }
            // all transitions leaving the rule start state need to leave blockStart instead
            while (atn.ruleToStartState[i].getNumberOfTransitions() > 0) {
                Transition transition = atn.ruleToStartState[i].removeTransition(atn.ruleToStartState[i].getNumberOfTransitions() - 1);
                bypassStart.addTransition(transition);
            }
            // link the new states
            atn.ruleToStartState[i].addTransition(new EpsilonTransition(bypassStart));
            bypassStop.addTransition(new EpsilonTransition(endState));
            ATNState matchState = new BasicState();
            atn.addState(matchState);
            matchState.addTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i]));
            bypassStart.addTransition(new EpsilonTransition(matchState));
        }
        if (deserializationOptions.isVerifyATN()) {
            // reverify after modification
            verifyATN(atn);
        }
    }
    return atn;
}
Also used : ArrayList(java.util.ArrayList) UUID(java.util.UUID) Pair(org.antlr.v4.runtime.misc.Pair) InvalidClassException(java.io.InvalidClassException) IntervalSet(org.antlr.v4.runtime.misc.IntervalSet)

Example 14 with LexerAction

use of org.antlr.v4.runtime.atn.LexerAction in project antlr4 by antlr.

the class LexerATNFactory method lexerCommand.

@Override
public Handle lexerCommand(GrammarAST ID) {
    LexerAction lexerAction = createLexerAction(ID, null);
    if (lexerAction != null) {
        return action(ID, lexerAction);
    }
    // fall back to standard action generation for the command
    ST cmdST = codegenTemplates.getInstanceOf("Lexer" + CharSupport.capitalize(ID.getText()) + "Command");
    if (cmdST == null) {
        g.tool.errMgr.grammarError(ErrorType.INVALID_LEXER_COMMAND, g.fileName, ID.token, ID.getText());
        return epsilon(ID);
    }
    if (cmdST.impl.formalArguments != null && cmdST.impl.formalArguments.containsKey("arg")) {
        g.tool.errMgr.grammarError(ErrorType.MISSING_LEXER_COMMAND_ARGUMENT, g.fileName, ID.token, ID.getText());
        return epsilon(ID);
    }
    return action(cmdST.render());
}
Also used : GrammarAST(org.antlr.v4.tool.ast.GrammarAST) ActionAST(org.antlr.v4.tool.ast.ActionAST) TerminalAST(org.antlr.v4.tool.ast.TerminalAST) RangeAST(org.antlr.v4.tool.ast.RangeAST) ST(org.stringtemplate.v4.ST) LexerAction(org.antlr.v4.runtime.atn.LexerAction)

Example 15 with LexerAction

use of org.antlr.v4.runtime.atn.LexerAction in project antlr4 by antlr.

the class LexerATNFactory method action.

@Override
public Handle action(ActionAST action) {
    int ruleIndex = currentRule.index;
    int actionIndex = g.lexerActions.get(action);
    LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex);
    return action(action, lexerAction);
}
Also used : LexerCustomAction(org.antlr.v4.runtime.atn.LexerCustomAction)

Aggregations

LexerAction (org.antlr.v4.runtime.atn.LexerAction)7 ArrayList (java.util.ArrayList)5 HashMap (java.util.HashMap)5 IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)5 ActionAST (org.antlr.v4.tool.ast.ActionAST)4 GrammarAST (org.antlr.v4.tool.ast.GrammarAST)4 RangeAST (org.antlr.v4.tool.ast.RangeAST)4 TerminalAST (org.antlr.v4.tool.ast.TerminalAST)4 ST (org.stringtemplate.v4.ST)4 ATNState (org.antlr.v4.runtime.atn.ATNState)3 ActionTransition (org.antlr.v4.runtime.atn.ActionTransition)3 LexerChannelAction (org.antlr.v4.runtime.atn.LexerChannelAction)3 LexerCustomAction (org.antlr.v4.runtime.atn.LexerCustomAction)3 LexerModeAction (org.antlr.v4.runtime.atn.LexerModeAction)3 LexerTypeAction (org.antlr.v4.runtime.atn.LexerTypeAction)3 RuleStartState (org.antlr.v4.runtime.atn.RuleStartState)3 IntegerList (org.antlr.v4.runtime.misc.IntegerList)3 InvalidClassException (java.io.InvalidClassException)2 LinkedHashMap (java.util.LinkedHashMap)2 Map (java.util.Map)2