Search in sources :

Example 1 with LexerTypeAction

use of org.antlr.v4.runtime.atn.LexerTypeAction in project antlr4 by antlr.

the class ATNSerializer method serialize.

/**
 * Serialize state descriptors, edge descriptors, and decision→state map
 *  into list of ints:
 *
 * 		grammar-type, (ANTLRParser.LEXER, ...)
 *  	max token type,
 *  	num states,
 *  	state-0-type ruleIndex, state-1-type ruleIndex, ... state-i-type ruleIndex optional-arg ...
 *  	num rules,
 *  	rule-1-start-state rule-1-args, rule-2-start-state  rule-2-args, ...
 *  	(args are token type,actionIndex in lexer else 0,0)
 *      num modes,
 *      mode-0-start-state, mode-1-start-state, ... (parser has 0 modes)
 *      num unicode-bmp-sets
 *      bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ...
 *      num unicode-smp-sets
 *      smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ...
 *	num total edges,
 *      src, trg, edge-type, edge arg1, optional edge arg2 (present always), ...
 *      num decisions,
 *      decision-0-start-state, decision-1-start-state, ...
 *
 *  Convenient to pack into unsigned shorts to make as Java string.
 */
public IntegerList serialize() {
    IntegerList data = new IntegerList();
    data.add(ATNDeserializer.SERIALIZED_VERSION);
    serializeUUID(data, ATNDeserializer.SERIALIZED_UUID);
    // convert grammar type to ATN const to avoid dependence on ANTLRParser
    data.add(atn.grammarType.ordinal());
    data.add(atn.maxTokenType);
    int nedges = 0;
    // Note that we use a LinkedHashMap as a set to
    // maintain insertion order while deduplicating
    // entries with the same key.
    Map<IntervalSet, Boolean> sets = new LinkedHashMap<>();
    // dump states, count edges and collect sets while doing so
    IntegerList nonGreedyStates = new IntegerList();
    IntegerList precedenceStates = new IntegerList();
    data.add(atn.states.size());
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            data.add(ATNState.INVALID_TYPE);
            continue;
        }
        int stateType = s.getStateType();
        if (s instanceof DecisionState && ((DecisionState) s).nonGreedy) {
            nonGreedyStates.add(s.stateNumber);
        }
        if (s instanceof RuleStartState && ((RuleStartState) s).isLeftRecursiveRule) {
            precedenceStates.add(s.stateNumber);
        }
        data.add(stateType);
        if (s.ruleIndex == -1) {
            data.add(Character.MAX_VALUE);
        } else {
            data.add(s.ruleIndex);
        }
        if (s.getStateType() == ATNState.LOOP_END) {
            data.add(((LoopEndState) s).loopBackState.stateNumber);
        } else if (s instanceof BlockStartState) {
            data.add(((BlockStartState) s).endState.stateNumber);
        }
        if (s.getStateType() != ATNState.RULE_STOP) {
            // the deserializer can trivially derive these edges, so there's no need to serialize them
            nedges += s.getNumberOfTransitions();
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            int edgeType = Transition.serializationTypes.get(t.getClass());
            if (edgeType == Transition.SET || edgeType == Transition.NOT_SET) {
                SetTransition st = (SetTransition) t;
                sets.put(st.set, true);
            }
        }
    }
    // non-greedy states
    data.add(nonGreedyStates.size());
    for (int i = 0; i < nonGreedyStates.size(); i++) {
        data.add(nonGreedyStates.get(i));
    }
    // precedence states
    data.add(precedenceStates.size());
    for (int i = 0; i < precedenceStates.size(); i++) {
        data.add(precedenceStates.get(i));
    }
    int nrules = atn.ruleToStartState.length;
    data.add(nrules);
    for (int r = 0; r < nrules; r++) {
        ATNState ruleStartState = atn.ruleToStartState[r];
        data.add(ruleStartState.stateNumber);
        if (atn.grammarType == ATNType.LEXER) {
            if (atn.ruleToTokenType[r] == Token.EOF) {
                data.add(Character.MAX_VALUE);
            } else {
                data.add(atn.ruleToTokenType[r]);
            }
        }
    }
    int nmodes = atn.modeToStartState.size();
    data.add(nmodes);
    if (nmodes > 0) {
        for (ATNState modeStartState : atn.modeToStartState) {
            data.add(modeStartState.stateNumber);
        }
    }
    List<IntervalSet> bmpSets = new ArrayList<>();
    List<IntervalSet> smpSets = new ArrayList<>();
    for (IntervalSet set : sets.keySet()) {
        if (!set.isNil() && set.getMaxElement() <= Character.MAX_VALUE) {
            bmpSets.add(set);
        } else {
            smpSets.add(set);
        }
    }
    serializeSets(data, bmpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            data.add(cp);
        }
    });
    serializeSets(data, smpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            serializeInt(data, cp);
        }
    });
    Map<IntervalSet, Integer> setIndices = new HashMap<>();
    int setIndex = 0;
    for (IntervalSet bmpSet : bmpSets) {
        setIndices.put(bmpSet, setIndex++);
    }
    for (IntervalSet smpSet : smpSets) {
        setIndices.put(smpSet, setIndex++);
    }
    data.add(nedges);
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            continue;
        }
        if (s.getStateType() == ATNState.RULE_STOP) {
            continue;
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            if (atn.states.get(t.target.stateNumber) == null) {
                throw new IllegalStateException("Cannot serialize a transition to a removed state.");
            }
            int src = s.stateNumber;
            int trg = t.target.stateNumber;
            int edgeType = Transition.serializationTypes.get(t.getClass());
            int arg1 = 0;
            int arg2 = 0;
            int arg3 = 0;
            switch(edgeType) {
                case Transition.RULE:
                    trg = ((RuleTransition) t).followState.stateNumber;
                    arg1 = ((RuleTransition) t).target.stateNumber;
                    arg2 = ((RuleTransition) t).ruleIndex;
                    arg3 = ((RuleTransition) t).precedence;
                    break;
                case Transition.PRECEDENCE:
                    PrecedencePredicateTransition ppt = (PrecedencePredicateTransition) t;
                    arg1 = ppt.precedence;
                    break;
                case Transition.PREDICATE:
                    PredicateTransition pt = (PredicateTransition) t;
                    arg1 = pt.ruleIndex;
                    arg2 = pt.predIndex;
                    arg3 = pt.isCtxDependent ? 1 : 0;
                    break;
                case Transition.RANGE:
                    arg1 = ((RangeTransition) t).from;
                    arg2 = ((RangeTransition) t).to;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ATOM:
                    arg1 = ((AtomTransition) t).label;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ACTION:
                    ActionTransition at = (ActionTransition) t;
                    arg1 = at.ruleIndex;
                    arg2 = at.actionIndex;
                    if (arg2 == -1) {
                        arg2 = 0xFFFF;
                    }
                    arg3 = at.isCtxDependent ? 1 : 0;
                    break;
                case Transition.SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.NOT_SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.WILDCARD:
                    break;
            }
            data.add(src);
            data.add(trg);
            data.add(edgeType);
            data.add(arg1);
            data.add(arg2);
            data.add(arg3);
        }
    }
    int ndecisions = atn.decisionToState.size();
    data.add(ndecisions);
    for (DecisionState decStartState : atn.decisionToState) {
        data.add(decStartState.stateNumber);
    }
    // 
    if (atn.grammarType == ATNType.LEXER) {
        data.add(atn.lexerActions.length);
        for (LexerAction action : atn.lexerActions) {
            data.add(action.getActionType().ordinal());
            switch(action.getActionType()) {
                case CHANNEL:
                    int channel = ((LexerChannelAction) action).getChannel();
                    data.add(channel != -1 ? channel : 0xFFFF);
                    data.add(0);
                    break;
                case CUSTOM:
                    int ruleIndex = ((LexerCustomAction) action).getRuleIndex();
                    int actionIndex = ((LexerCustomAction) action).getActionIndex();
                    data.add(ruleIndex != -1 ? ruleIndex : 0xFFFF);
                    data.add(actionIndex != -1 ? actionIndex : 0xFFFF);
                    break;
                case MODE:
                    int mode = ((LexerModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case MORE:
                    data.add(0);
                    data.add(0);
                    break;
                case POP_MODE:
                    data.add(0);
                    data.add(0);
                    break;
                case PUSH_MODE:
                    mode = ((LexerPushModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case SKIP:
                    data.add(0);
                    data.add(0);
                    break;
                case TYPE:
                    int type = ((LexerTypeAction) action).getType();
                    data.add(type != -1 ? type : 0xFFFF);
                    data.add(0);
                    break;
                default:
                    String message = String.format(Locale.getDefault(), "The specified lexer action type %s is not valid.", action.getActionType());
                    throw new IllegalArgumentException(message);
            }
        }
    }
    // don't adjust the first value since that's the version number
    for (int i = 1; i < data.size(); i++) {
        if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) {
            throw new UnsupportedOperationException("Serialized ATN data element " + data.get(i) + " element " + i + " out of range " + (int) Character.MIN_VALUE + ".." + (int) Character.MAX_VALUE);
        }
        int value = (data.get(i) + 2) & 0xFFFF;
        data.set(i, value);
    }
    return data;
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) IntegerList(org.antlr.v4.runtime.misc.IntegerList) IntervalSet(org.antlr.v4.runtime.misc.IntervalSet)

Example 2 with LexerTypeAction

use of org.antlr.v4.runtime.atn.LexerTypeAction in project antlr4 by antlr.

the class SwiftTarget method serializeTojson.

// added by janyou -->
public String serializeTojson(ATN atn) {
    JsonObjectBuilder builder = Json.createObjectBuilder();
    builder.add("version", ATNDeserializer.SERIALIZED_VERSION);
    builder.add("uuid", ATNDeserializer.SERIALIZED_UUID.toString());
    // convert grammar type to ATN const to avoid dependence on ANTLRParser
    builder.add("grammarType", atn.grammarType.ordinal());
    builder.add("maxTokenType", atn.maxTokenType);
    // states
    int nedges = 0;
    Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>();
    List<IntervalSet> sets = new ArrayList<IntervalSet>();
    JsonArrayBuilder statesBuilder = Json.createArrayBuilder();
    IntegerList nonGreedyStates = new IntegerList();
    IntegerList precedenceStates = new IntegerList();
    for (ATNState s : atn.states) {
        JsonObjectBuilder stateBuilder = Json.createObjectBuilder();
        if (s == null) {
            // might be optimized away
            statesBuilder.addNull();
            continue;
        }
        int stateType = s.getStateType();
        stateBuilder.add("stateType", stateType);
        // stateBuilder.add("stateNumber",s.stateNumber);
        stateBuilder.add("ruleIndex", s.ruleIndex);
        if (s instanceof DecisionState && ((DecisionState) s).nonGreedy) {
            nonGreedyStates.add(s.stateNumber);
        }
        if (s instanceof RuleStartState && ((RuleStartState) s).isLeftRecursiveRule) {
            precedenceStates.add(s.stateNumber);
        }
        if (s.getStateType() == ATNState.LOOP_END) {
            stateBuilder.add("detailStateNumber", ((LoopEndState) s).loopBackState.stateNumber);
        } else if (s instanceof BlockStartState) {
            stateBuilder.add("detailStateNumber", ((BlockStartState) s).endState.stateNumber);
        }
        if (s.getStateType() != ATNState.RULE_STOP) {
            // the deserializer can trivially derive these edges, so there's no need to serialize them
            nedges += s.getNumberOfTransitions();
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            int edgeType = Transition.serializationTypes.get(t.getClass());
            if (edgeType == Transition.SET || edgeType == Transition.NOT_SET) {
                SetTransition st = (SetTransition) t;
                if (!setIndices.containsKey(st.set)) {
                    sets.add(st.set);
                    setIndices.put(st.set, sets.size() - 1);
                }
            }
        }
        statesBuilder.add(stateBuilder);
    }
    builder.add("states", statesBuilder);
    // non-greedy states
    JsonArrayBuilder nonGreedyStatesBuilder = Json.createArrayBuilder();
    for (int i = 0; i < nonGreedyStates.size(); i++) {
        nonGreedyStatesBuilder.add(nonGreedyStates.get(i));
    }
    builder.add("nonGreedyStates", nonGreedyStatesBuilder);
    // precedence states
    JsonArrayBuilder precedenceStatesBuilder = Json.createArrayBuilder();
    for (int i = 0; i < precedenceStates.size(); i++) {
        precedenceStatesBuilder.add(precedenceStates.get(i));
    }
    builder.add("precedenceStates", precedenceStatesBuilder);
    JsonArrayBuilder ruleToStartStateBuilder = Json.createArrayBuilder();
    int nrules = atn.ruleToStartState.length;
    for (int r = 0; r < nrules; r++) {
        JsonObjectBuilder stateBuilder = Json.createObjectBuilder();
        ATNState ruleStartState = atn.ruleToStartState[r];
        stateBuilder.add("stateNumber", ruleStartState.stateNumber);
        if (atn.grammarType == ATNType.LEXER) {
            // if (atn.ruleToTokenType[r] == Token.EOF) {
            // //data.add(Character.MAX_VALUE);
            // stateBuilder.add("ruleToTokenType",-1);
            // }
            // else {
            // //data.add(atn.ruleToTokenType[r]);
            stateBuilder.add("ruleToTokenType", atn.ruleToTokenType[r]);
        // }
        }
        ruleToStartStateBuilder.add(stateBuilder);
    }
    builder.add("ruleToStartState", ruleToStartStateBuilder);
    JsonArrayBuilder modeToStartStateBuilder = Json.createArrayBuilder();
    int nmodes = atn.modeToStartState.size();
    if (nmodes > 0) {
        for (ATNState modeStartState : atn.modeToStartState) {
            modeToStartStateBuilder.add(modeStartState.stateNumber);
        }
    }
    builder.add("modeToStartState", modeToStartStateBuilder);
    JsonArrayBuilder nsetsBuilder = Json.createArrayBuilder();
    int nsets = sets.size();
    // data.add(nsets);
    builder.add("nsets", nsets);
    for (IntervalSet set : sets) {
        JsonObjectBuilder setBuilder = Json.createObjectBuilder();
        boolean containsEof = set.contains(Token.EOF);
        if (containsEof && set.getIntervals().get(0).b == Token.EOF) {
            // data.add(set.getIntervals().size() - 1);
            setBuilder.add("size", set.getIntervals().size() - 1);
        } else {
            // data.add(set.getIntervals().size());
            setBuilder.add("size", set.getIntervals().size());
        }
        setBuilder.add("containsEof", containsEof ? 1 : 0);
        JsonArrayBuilder IntervalsBuilder = Json.createArrayBuilder();
        for (Interval I : set.getIntervals()) {
            JsonObjectBuilder IntervalBuilder = Json.createObjectBuilder();
            if (I.a == Token.EOF) {
                if (I.b == Token.EOF) {
                    continue;
                } else {
                    IntervalBuilder.add("a", 0);
                // data.add(0);
                }
            } else {
                IntervalBuilder.add("a", I.a);
            // data.add(I.a);
            }
            IntervalBuilder.add("b", I.b);
            IntervalsBuilder.add(IntervalBuilder);
        }
        setBuilder.add("Intervals", IntervalsBuilder);
        nsetsBuilder.add(setBuilder);
    }
    builder.add("IntervalSet", nsetsBuilder);
    // builder.add("nedges",nedges);
    JsonArrayBuilder allTransitionsBuilder = Json.createArrayBuilder();
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            continue;
        }
        if (s.getStateType() == ATNState.RULE_STOP) {
            continue;
        }
        JsonArrayBuilder transitionsBuilder = Json.createArrayBuilder();
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            JsonObjectBuilder transitionBuilder = Json.createObjectBuilder();
            Transition t = s.transition(i);
            if (atn.states.get(t.target.stateNumber) == null) {
                throw new IllegalStateException("Cannot serialize a transition to a removed state.");
            }
            int src = s.stateNumber;
            int trg = t.target.stateNumber;
            int edgeType = Transition.serializationTypes.get(t.getClass());
            int arg1 = 0;
            int arg2 = 0;
            int arg3 = 0;
            switch(edgeType) {
                case Transition.RULE:
                    trg = ((RuleTransition) t).followState.stateNumber;
                    arg1 = ((RuleTransition) t).target.stateNumber;
                    arg2 = ((RuleTransition) t).ruleIndex;
                    arg3 = ((RuleTransition) t).precedence;
                    break;
                case Transition.PRECEDENCE:
                    PrecedencePredicateTransition ppt = (PrecedencePredicateTransition) t;
                    arg1 = ppt.precedence;
                    break;
                case Transition.PREDICATE:
                    PredicateTransition pt = (PredicateTransition) t;
                    arg1 = pt.ruleIndex;
                    arg2 = pt.predIndex;
                    arg3 = pt.isCtxDependent ? 1 : 0;
                    break;
                case Transition.RANGE:
                    arg1 = ((RangeTransition) t).from;
                    arg2 = ((RangeTransition) t).to;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ATOM:
                    arg1 = ((AtomTransition) t).label;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ACTION:
                    ActionTransition at = (ActionTransition) t;
                    arg1 = at.ruleIndex;
                    arg2 = at.actionIndex;
                    // if (arg2 == -1) {
                    // arg2 = 0xFFFF;
                    // }
                    arg3 = at.isCtxDependent ? 1 : 0;
                    break;
                case Transition.SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.NOT_SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.WILDCARD:
                    break;
            }
            transitionBuilder.add("src", src);
            transitionBuilder.add("trg", trg);
            transitionBuilder.add("edgeType", edgeType);
            transitionBuilder.add("arg1", arg1);
            transitionBuilder.add("arg2", arg2);
            transitionBuilder.add("arg3", arg3);
            transitionsBuilder.add(transitionBuilder);
        }
        allTransitionsBuilder.add(transitionsBuilder);
    }
    builder.add("allTransitionsBuilder", allTransitionsBuilder);
    int ndecisions = atn.decisionToState.size();
    // data.add(ndecisions);
    JsonArrayBuilder decisionToStateBuilder = Json.createArrayBuilder();
    for (DecisionState decStartState : atn.decisionToState) {
        // data.add(decStartState.stateNumber);
        decisionToStateBuilder.add(decStartState.stateNumber);
    }
    builder.add("decisionToState", decisionToStateBuilder);
    // 
    // LEXER ACTIONS
    // 
    JsonArrayBuilder lexerActionsBuilder = Json.createArrayBuilder();
    if (atn.grammarType == ATNType.LEXER) {
        // data.add(atn.lexerActions.length);
        for (LexerAction action : atn.lexerActions) {
            JsonObjectBuilder lexerActionBuilder = Json.createObjectBuilder();
            lexerActionBuilder.add("actionType", action.getActionType().ordinal());
            // data.add(action.getActionType().ordinal());
            switch(action.getActionType()) {
                case CHANNEL:
                    int channel = ((LexerChannelAction) action).getChannel();
                    lexerActionBuilder.add("a", channel);
                    lexerActionBuilder.add("b", 0);
                    break;
                case CUSTOM:
                    int ruleIndex = ((LexerCustomAction) action).getRuleIndex();
                    int actionIndex = ((LexerCustomAction) action).getActionIndex();
                    lexerActionBuilder.add("a", ruleIndex);
                    lexerActionBuilder.add("b", actionIndex);
                    break;
                case MODE:
                    int mode = ((LexerModeAction) action).getMode();
                    lexerActionBuilder.add("a", mode);
                    lexerActionBuilder.add("b", 0);
                    break;
                case MORE:
                    lexerActionBuilder.add("a", 0);
                    lexerActionBuilder.add("b", 0);
                    break;
                case POP_MODE:
                    lexerActionBuilder.add("a", 0);
                    lexerActionBuilder.add("b", 0);
                    break;
                case PUSH_MODE:
                    mode = ((LexerPushModeAction) action).getMode();
                    lexerActionBuilder.add("a", mode);
                    lexerActionBuilder.add("b", 0);
                    break;
                case SKIP:
                    lexerActionBuilder.add("a", 0);
                    lexerActionBuilder.add("b", 0);
                    break;
                case TYPE:
                    int type = ((LexerTypeAction) action).getType();
                    lexerActionBuilder.add("a", type);
                    lexerActionBuilder.add("b", 0);
                    break;
                default:
                    String message = String.format(Locale.getDefault(), "The specified lexer action type %s is not valid.", action.getActionType());
                    throw new IllegalArgumentException(message);
            }
            lexerActionsBuilder.add(lexerActionBuilder);
        }
    }
    builder.add("lexerActions", lexerActionsBuilder);
    // don't adjust the first value since that's the version number
    // for (int i = 1; i < data.size(); i++) {
    // if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) {
    // throw new UnsupportedOperationException("Serialized ATN data element out of range.");
    // }
    // 
    // int value = (data.get(i) + 2) & 0xFFFF;
    // data.set(i, value);
    // }
    JsonObject data = builder.build();
    // System.out.print(data.toString());
    return data.toString();
}
Also used : HashMap(java.util.HashMap) LexerTypeAction(org.antlr.v4.runtime.atn.LexerTypeAction) ArrayList(java.util.ArrayList) SetTransition(org.antlr.v4.runtime.atn.SetTransition) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) JsonObject(javax.json.JsonObject) ATNState(org.antlr.v4.runtime.atn.ATNState) IntegerList(org.antlr.v4.runtime.misc.IntegerList) LexerModeAction(org.antlr.v4.runtime.atn.LexerModeAction) JsonArrayBuilder(javax.json.JsonArrayBuilder) JsonObjectBuilder(javax.json.JsonObjectBuilder) LoopEndState(org.antlr.v4.runtime.atn.LoopEndState) LexerChannelAction(org.antlr.v4.runtime.atn.LexerChannelAction) LexerAction(org.antlr.v4.runtime.atn.LexerAction) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) PredicateTransition(org.antlr.v4.runtime.atn.PredicateTransition) DecisionState(org.antlr.v4.runtime.atn.DecisionState) ActionTransition(org.antlr.v4.runtime.atn.ActionTransition) RuleStartState(org.antlr.v4.runtime.atn.RuleStartState) IntervalSet(org.antlr.v4.runtime.misc.IntervalSet) RuleTransition(org.antlr.v4.runtime.atn.RuleTransition) RuleTransition(org.antlr.v4.runtime.atn.RuleTransition) PrecedencePredicateTransition(org.antlr.v4.runtime.atn.PrecedencePredicateTransition) PredicateTransition(org.antlr.v4.runtime.atn.PredicateTransition) ActionTransition(org.antlr.v4.runtime.atn.ActionTransition) Transition(org.antlr.v4.runtime.atn.Transition) AtomTransition(org.antlr.v4.runtime.atn.AtomTransition) SetTransition(org.antlr.v4.runtime.atn.SetTransition) RangeTransition(org.antlr.v4.runtime.atn.RangeTransition) BlockStartState(org.antlr.v4.runtime.atn.BlockStartState) LexerCustomAction(org.antlr.v4.runtime.atn.LexerCustomAction) Interval(org.antlr.v4.runtime.misc.Interval)

Example 3 with LexerTypeAction

use of org.antlr.v4.runtime.atn.LexerTypeAction in project antlr4 by tunnelvisionlabs.

the class ATNSerializer method serialize.

/**
 * Serialize state descriptors, edge descriptors, and decision&rarr;state map
 *  into list of ints:
 *
 * 		grammar-type, (ANTLRParser.LEXER, ...)
 *  	max token type,
 *  	num states,
 *  	state-0-type ruleIndex, state-1-type ruleIndex, ... state-i-type ruleIndex optional-arg ...
 *  	num rules,
 *  	rule-1-start-state rule-1-args, rule-2-start-state  rule-2-args, ...
 *  	(args are token type,actionIndex in lexer else 0,0)
 *      num modes,
 *      mode-0-start-state, mode-1-start-state, ... (parser has 0 modes)
 *      num unicode-bmp-sets
 *      bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ...
 *      num unicode-smp-sets
 *      smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ...
 *	num total edges,
 *      src, trg, edge-type, edge arg1, optional edge arg2 (present always), ...
 *      num decisions,
 *      decision-0-start-state, decision-1-start-state, ...
 *
 *  Convenient to pack into unsigned shorts to make as Java string.
 */
public IntegerList serialize() {
    IntegerList data = new IntegerList();
    data.add(ATNDeserializer.SERIALIZED_VERSION);
    serializeUUID(data, ATNDeserializer.SERIALIZED_UUID);
    // convert grammar type to ATN const to avoid dependence on ANTLRParser
    data.add(atn.grammarType.ordinal());
    data.add(atn.maxTokenType);
    int nedges = 0;
    // Note that we use a LinkedHashMap as a set to
    // maintain insertion order while deduplicating
    // entries with the same key.
    Map<IntervalSet, Boolean> sets = new LinkedHashMap<IntervalSet, Boolean>();
    // dump states, count edges and collect sets while doing so
    IntegerList nonGreedyStates = new IntegerList();
    IntegerList sllStates = new IntegerList();
    IntegerList precedenceStates = new IntegerList();
    data.add(atn.states.size());
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            data.add(ATNState.INVALID_TYPE);
            continue;
        }
        int stateType = s.getStateType();
        if (s instanceof DecisionState) {
            DecisionState decisionState = (DecisionState) s;
            if (decisionState.nonGreedy) {
                nonGreedyStates.add(s.stateNumber);
            }
            if (decisionState.sll) {
                sllStates.add(s.stateNumber);
            }
        }
        if (s instanceof RuleStartState && ((RuleStartState) s).isPrecedenceRule) {
            precedenceStates.add(s.stateNumber);
        }
        data.add(stateType);
        if (s.ruleIndex == -1) {
            data.add(Character.MAX_VALUE);
        } else {
            data.add(s.ruleIndex);
        }
        if (s.getStateType() == ATNState.LOOP_END) {
            data.add(((LoopEndState) s).loopBackState.stateNumber);
        } else if (s instanceof BlockStartState) {
            data.add(((BlockStartState) s).endState.stateNumber);
        }
        if (s.getStateType() != ATNState.RULE_STOP) {
            // the deserializer can trivially derive these edges, so there's no need to serialize them
            nedges += s.getNumberOfTransitions();
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            int edgeType = Transition.serializationTypes.get(t.getClass());
            if (edgeType == Transition.SET || edgeType == Transition.NOT_SET) {
                SetTransition st = (SetTransition) t;
                sets.put(st.set, true);
            }
        }
    }
    // non-greedy states
    data.add(nonGreedyStates.size());
    for (int i = 0; i < nonGreedyStates.size(); i++) {
        data.add(nonGreedyStates.get(i));
    }
    // SLL decisions
    data.add(sllStates.size());
    for (int i = 0; i < sllStates.size(); i++) {
        data.add(sllStates.get(i));
    }
    // precedence states
    data.add(precedenceStates.size());
    for (int i = 0; i < precedenceStates.size(); i++) {
        data.add(precedenceStates.get(i));
    }
    int nrules = atn.ruleToStartState.length;
    data.add(nrules);
    for (int r = 0; r < nrules; r++) {
        ATNState ruleStartState = atn.ruleToStartState[r];
        data.add(ruleStartState.stateNumber);
        boolean leftFactored = ruleNames.get(ruleStartState.ruleIndex).indexOf(ATNSimulator.RULE_VARIANT_DELIMITER) >= 0;
        data.add(leftFactored ? 1 : 0);
        if (atn.grammarType == ATNType.LEXER) {
            if (atn.ruleToTokenType[r] == Token.EOF) {
                data.add(Character.MAX_VALUE);
            } else {
                data.add(atn.ruleToTokenType[r]);
            }
        }
    }
    int nmodes = atn.modeToStartState.size();
    data.add(nmodes);
    if (nmodes > 0) {
        for (ATNState modeStartState : atn.modeToStartState) {
            data.add(modeStartState.stateNumber);
        }
    }
    List<IntervalSet> bmpSets = new ArrayList<IntervalSet>();
    List<IntervalSet> smpSets = new ArrayList<IntervalSet>();
    for (IntervalSet set : sets.keySet()) {
        if (set.getMaxElement() <= Character.MAX_VALUE) {
            bmpSets.add(set);
        } else {
            smpSets.add(set);
        }
    }
    serializeSets(data, bmpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            data.add(cp);
        }
    });
    serializeSets(data, smpSets, new CodePointSerializer() {

        @Override
        public void serializeCodePoint(IntegerList data, int cp) {
            serializeInt(data, cp);
        }
    });
    Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>();
    int setIndex = 0;
    for (IntervalSet bmpSet : bmpSets) {
        setIndices.put(bmpSet, setIndex++);
    }
    for (IntervalSet smpSet : smpSets) {
        setIndices.put(smpSet, setIndex++);
    }
    data.add(nedges);
    for (ATNState s : atn.states) {
        if (s == null) {
            // might be optimized away
            continue;
        }
        if (s.getStateType() == ATNState.RULE_STOP) {
            continue;
        }
        for (int i = 0; i < s.getNumberOfTransitions(); i++) {
            Transition t = s.transition(i);
            if (atn.states.get(t.target.stateNumber) == null) {
                throw new IllegalStateException("Cannot serialize a transition to a removed state.");
            }
            int src = s.stateNumber;
            int trg = t.target.stateNumber;
            int edgeType = Transition.serializationTypes.get(t.getClass());
            int arg1 = 0;
            int arg2 = 0;
            int arg3 = 0;
            switch(edgeType) {
                case Transition.RULE:
                    trg = ((RuleTransition) t).followState.stateNumber;
                    arg1 = ((RuleTransition) t).target.stateNumber;
                    arg2 = ((RuleTransition) t).ruleIndex;
                    arg3 = ((RuleTransition) t).precedence;
                    break;
                case Transition.PRECEDENCE:
                    PrecedencePredicateTransition ppt = (PrecedencePredicateTransition) t;
                    arg1 = ppt.precedence;
                    break;
                case Transition.PREDICATE:
                    PredicateTransition pt = (PredicateTransition) t;
                    arg1 = pt.ruleIndex;
                    arg2 = pt.predIndex;
                    arg3 = pt.isCtxDependent ? 1 : 0;
                    break;
                case Transition.RANGE:
                    arg1 = ((RangeTransition) t).from;
                    arg2 = ((RangeTransition) t).to;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ATOM:
                    arg1 = ((AtomTransition) t).label;
                    if (arg1 == Token.EOF) {
                        arg1 = 0;
                        arg3 = 1;
                    }
                    break;
                case Transition.ACTION:
                    ActionTransition at = (ActionTransition) t;
                    arg1 = at.ruleIndex;
                    arg2 = at.actionIndex;
                    if (arg2 == -1) {
                        arg2 = 0xFFFF;
                    }
                    arg3 = at.isCtxDependent ? 1 : 0;
                    break;
                case Transition.SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.NOT_SET:
                    arg1 = setIndices.get(((SetTransition) t).set);
                    break;
                case Transition.WILDCARD:
                    break;
            }
            data.add(src);
            data.add(trg);
            data.add(edgeType);
            data.add(arg1);
            data.add(arg2);
            data.add(arg3);
        }
    }
    int ndecisions = atn.decisionToState.size();
    data.add(ndecisions);
    for (DecisionState decStartState : atn.decisionToState) {
        data.add(decStartState.stateNumber);
    }
    // 
    if (atn.grammarType == ATNType.LEXER) {
        data.add(atn.lexerActions.length);
        for (LexerAction action : atn.lexerActions) {
            data.add(action.getActionType().ordinal());
            switch(action.getActionType()) {
                case CHANNEL:
                    int channel = ((LexerChannelAction) action).getChannel();
                    data.add(channel != -1 ? channel : 0xFFFF);
                    data.add(0);
                    break;
                case CUSTOM:
                    int ruleIndex = ((LexerCustomAction) action).getRuleIndex();
                    int actionIndex = ((LexerCustomAction) action).getActionIndex();
                    data.add(ruleIndex != -1 ? ruleIndex : 0xFFFF);
                    data.add(actionIndex != -1 ? actionIndex : 0xFFFF);
                    break;
                case MODE:
                    int mode = ((LexerModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case MORE:
                    data.add(0);
                    data.add(0);
                    break;
                case POP_MODE:
                    data.add(0);
                    data.add(0);
                    break;
                case PUSH_MODE:
                    mode = ((LexerPushModeAction) action).getMode();
                    data.add(mode != -1 ? mode : 0xFFFF);
                    data.add(0);
                    break;
                case SKIP:
                    data.add(0);
                    data.add(0);
                    break;
                case TYPE:
                    int type = ((LexerTypeAction) action).getType();
                    data.add(type != -1 ? type : 0xFFFF);
                    data.add(0);
                    break;
                default:
                    String message = String.format(Locale.getDefault(), "The specified lexer action type %s is not valid.", action.getActionType());
                    throw new IllegalArgumentException(message);
            }
        }
    }
    // don't adjust the first value since that's the version number
    for (int i = 1; i < data.size(); i++) {
        if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) {
            throw new UnsupportedOperationException("Serialized ATN data element " + data.get(i) + " element " + i + " out of range " + (int) Character.MIN_VALUE + ".." + (int) Character.MAX_VALUE);
        }
        int value = (data.get(i) + 2) & 0xFFFF;
        data.set(i, value);
    }
    return data;
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) IntegerList(org.antlr.v4.runtime.misc.IntegerList) IntervalSet(org.antlr.v4.runtime.misc.IntervalSet)

Example 4 with LexerTypeAction

use of org.antlr.v4.runtime.atn.LexerTypeAction in project antlr4 by tunnelvisionlabs.

the class LexerATNFactory method createLexerAction.

@Nullable
private LexerAction createLexerAction(@NotNull GrammarAST ID, @Nullable GrammarAST arg) {
    String command = ID.getText();
    checkCommands(command, ID.getToken());
    if ("skip".equals(command) && arg == null) {
        return LexerSkipAction.INSTANCE;
    } else if ("more".equals(command) && arg == null) {
        return LexerMoreAction.INSTANCE;
    } else if ("popMode".equals(command) && arg == null) {
        return LexerPopModeAction.INSTANCE;
    } else if ("mode".equals(command) && arg != null) {
        String modeName = arg.getText();
        Integer mode = getModeConstantValue(modeName, arg.getToken());
        if (mode == null) {
            return null;
        }
        return new LexerModeAction(mode);
    } else if ("pushMode".equals(command) && arg != null) {
        String modeName = arg.getText();
        Integer mode = getModeConstantValue(modeName, arg.getToken());
        if (mode == null) {
            return null;
        }
        return new LexerPushModeAction(mode);
    } else if ("type".equals(command) && arg != null) {
        String typeName = arg.getText();
        Integer type = getTokenConstantValue(typeName, arg.getToken());
        if (type == null) {
            return null;
        }
        return new LexerTypeAction(type);
    } else if ("channel".equals(command) && arg != null) {
        String channelName = arg.getText();
        Integer channel = getChannelConstantValue(channelName, arg.getToken());
        if (channel == null) {
            return null;
        }
        return new LexerChannelAction(channel);
    } else {
        return null;
    }
}
Also used : LexerPushModeAction(org.antlr.v4.runtime.atn.LexerPushModeAction) LexerChannelAction(org.antlr.v4.runtime.atn.LexerChannelAction) LexerTypeAction(org.antlr.v4.runtime.atn.LexerTypeAction) LexerModeAction(org.antlr.v4.runtime.atn.LexerModeAction) Nullable(org.antlr.v4.runtime.misc.Nullable)

Example 5 with LexerTypeAction

use of org.antlr.v4.runtime.atn.LexerTypeAction in project antlr4 by antlr.

the class LexerATNFactory method createLexerAction.

private LexerAction createLexerAction(GrammarAST ID, GrammarAST arg) {
    String command = ID.getText();
    checkCommands(command, ID.getToken());
    if ("skip".equals(command) && arg == null) {
        return LexerSkipAction.INSTANCE;
    } else if ("more".equals(command) && arg == null) {
        return LexerMoreAction.INSTANCE;
    } else if ("popMode".equals(command) && arg == null) {
        return LexerPopModeAction.INSTANCE;
    } else if ("mode".equals(command) && arg != null) {
        String modeName = arg.getText();
        Integer mode = getModeConstantValue(modeName, arg.getToken());
        if (mode == null) {
            return null;
        }
        return new LexerModeAction(mode);
    } else if ("pushMode".equals(command) && arg != null) {
        String modeName = arg.getText();
        Integer mode = getModeConstantValue(modeName, arg.getToken());
        if (mode == null) {
            return null;
        }
        return new LexerPushModeAction(mode);
    } else if ("type".equals(command) && arg != null) {
        String typeName = arg.getText();
        Integer type = getTokenConstantValue(typeName, arg.getToken());
        if (type == null) {
            return null;
        }
        return new LexerTypeAction(type);
    } else if ("channel".equals(command) && arg != null) {
        String channelName = arg.getText();
        Integer channel = getChannelConstantValue(channelName, arg.getToken());
        if (channel == null) {
            return null;
        }
        return new LexerChannelAction(channel);
    } else {
        return null;
    }
}
Also used : LexerPushModeAction(org.antlr.v4.runtime.atn.LexerPushModeAction) LexerChannelAction(org.antlr.v4.runtime.atn.LexerChannelAction) LexerTypeAction(org.antlr.v4.runtime.atn.LexerTypeAction) LexerModeAction(org.antlr.v4.runtime.atn.LexerModeAction)

Aggregations

ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 LexerChannelAction (org.antlr.v4.runtime.atn.LexerChannelAction)3 LexerModeAction (org.antlr.v4.runtime.atn.LexerModeAction)3 LexerTypeAction (org.antlr.v4.runtime.atn.LexerTypeAction)3 IntegerList (org.antlr.v4.runtime.misc.IntegerList)3 IntervalSet (org.antlr.v4.runtime.misc.IntervalSet)3 LinkedHashMap (java.util.LinkedHashMap)2 LexerPushModeAction (org.antlr.v4.runtime.atn.LexerPushModeAction)2 JsonArrayBuilder (javax.json.JsonArrayBuilder)1 JsonObject (javax.json.JsonObject)1 JsonObjectBuilder (javax.json.JsonObjectBuilder)1 ATNState (org.antlr.v4.runtime.atn.ATNState)1 ActionTransition (org.antlr.v4.runtime.atn.ActionTransition)1 AtomTransition (org.antlr.v4.runtime.atn.AtomTransition)1 BlockStartState (org.antlr.v4.runtime.atn.BlockStartState)1 DecisionState (org.antlr.v4.runtime.atn.DecisionState)1 LexerAction (org.antlr.v4.runtime.atn.LexerAction)1 LexerCustomAction (org.antlr.v4.runtime.atn.LexerCustomAction)1 LoopEndState (org.antlr.v4.runtime.atn.LoopEndState)1