Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 51 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LeftRecursiveRuleTransformer method parseArtificialRule.

public RuleAST parseArtificialRule(final Grammar g, String ruleText) {
    ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(lexer.getCharStream());
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    lexer.tokens = tokens;
    ToolANTLRParser p = new ToolANTLRParser(tokens, tool);
    p.setTreeAdaptor(adaptor);
    Token ruleStart = null;
    try {
        ParserRuleReturnScope r = p.rule();
        RuleAST tree = (RuleAST) r.getTree();
        ruleStart = (Token) r.getStart();
        GrammarTransformPipeline.setGrammarPtr(g, tree);
        GrammarTransformPipeline.augmentTokensWithOriginalPosition(g, tree);
        return tree;
    } catch (Exception e) {
        tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e, ruleStart, "error parsing rule created during left-recursion detection: " + ruleText);
    }
    return null;
}

Example 52 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ATNOptimizer method optimizeSets.

private static void optimizeSets(Grammar g, ATN atn) {
    if (g.isParser()) {
        // parser codegen doesn't currently support SetTransition
        return;
    }
    int removedStates = 0;
    List<DecisionState> decisions = atn.decisionToState;
    for (DecisionState decision : decisions) {
        if (decision.ruleIndex >= 0) {
            Rule rule = g.getRule(decision.ruleIndex);
            if (Character.isLowerCase(rule.name.charAt(0))) {
                // parser codegen doesn't currently support SetTransition
                continue;
            }
        }
        IntervalSet setTransitions = new IntervalSet();
        for (int i = 0; i < decision.getNumberOfTransitions(); i++) {
            Transition epsTransition = decision.transition(i);
            if (!(epsTransition instanceof EpsilonTransition)) {
                continue;
            }
            if (epsTransition.target.getNumberOfTransitions() != 1) {
                continue;
            }
            Transition transition = epsTransition.target.transition(0);
            if (!(transition.target instanceof BlockEndState)) {
                continue;
            }
            if (transition instanceof NotSetTransition) {
                // TODO: not yet implemented
                continue;
            }
            if (transition instanceof AtomTransition || transition instanceof RangeTransition || transition instanceof SetTransition) {
                setTransitions.add(i);
            }
        }
        // due to min alt resolution policies, can only collapse sequential alts
        for (int i = setTransitions.getIntervals().size() - 1; i >= 0; i--) {
            Interval interval = setTransitions.getIntervals().get(i);
            if (interval.length() <= 1) {
                continue;
            }
            ATNState blockEndState = decision.transition(interval.a).target.transition(0).target;
            IntervalSet matchSet = new IntervalSet();
            for (int j = interval.a; j <= interval.b; j++) {
                Transition matchTransition = decision.transition(j).target.transition(0);
                if (matchTransition instanceof NotSetTransition) {
                    throw new UnsupportedOperationException("Not yet implemented.");
                }
                IntervalSet set = matchTransition.label();
                List<Interval> intervals = set.getIntervals();
                int n = intervals.size();
                for (int k = 0; k < n; k++) {
                    Interval setInterval = intervals.get(k);
                    int a = setInterval.a;
                    int b = setInterval.b;
                    if (a != -1 && b != -1) {
                        for (int v = a; v <= b; v++) {
                            if (matchSet.contains(v)) {
                                // TODO: Token is missing (i.e. position in source will not be displayed).
                                g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, null, CharSupport.getANTLRCharLiteralForChar(v), CharSupport.getIntervalSetEscapedString(matchSet));
                                break;
                            }
                        }
                    }
                }
                matchSet.addAll(set);
            }
            Transition newTransition;
            if (matchSet.getIntervals().size() == 1) {
                if (matchSet.size() == 1) {
                    newTransition = CodePointTransitions.createWithCodePoint(blockEndState, matchSet.getMinElement());
                } else {
                    Interval matchInterval = matchSet.getIntervals().get(0);
                    newTransition = CodePointTransitions.createWithCodePointRange(blockEndState, matchInterval.a, matchInterval.b);
                }
            } else {
                newTransition = new SetTransition(blockEndState, matchSet);
            }
            decision.transition(interval.a).target.setTransition(0, newTransition);
            for (int j = interval.a + 1; j <= interval.b; j++) {
                Transition removed = decision.removeTransition(interval.a + 1);
                atn.removeState(removed.target);
                removedStates++;
            }
        }
    }
//		System.out.println("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}

Example 53 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ActionTranslator method getTokenPropertyRef.

TokenPropertyRef getTokenPropertyRef(Token x, Token y) {
    try {
        Class<? extends TokenPropertyRef> c = tokenPropToModelMap.get(y.getText());
        Constructor<? extends TokenPropertyRef> ctor = c.getConstructor(StructDecl.class, String.class);
        TokenPropertyRef ref = ctor.newInstance(nodeContext, getTokenLabel(x.getText()));
        return ref;
    } catch (Exception e) {
        factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e);
    }
    return null;
}

Example 54 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ActionTranslator method attr.

@Override
public void attr(String expr, Token x) {
    gen.g.tool.log("action-translator", "attr " + x);
    Attribute a = node.resolver.resolveToAttribute(x.getText(), node);
    if (a != null) {
        switch(a.dict.type) {
            case ARG:
                chunks.add(new ArgRef(nodeContext, x.getText()));
                break;
            case RET:
                chunks.add(new RetValueRef(rf.ruleCtx, x.getText()));
                break;
            case LOCAL:
                chunks.add(new LocalRef(nodeContext, x.getText()));
                break;
            case PREDEFINED_RULE:
                chunks.add(getRulePropertyRef(x));
                break;
        }
    }
    if (node.resolver.resolvesToToken(x.getText(), node)) {
        // $label
        chunks.add(new TokenRef(nodeContext, getTokenLabel(x.getText())));
        return;
    }
    if (node.resolver.resolvesToLabel(x.getText(), node)) {
        // $x for x=ID etc...
        chunks.add(new LabelRef(nodeContext, getTokenLabel(x.getText())));
        return;
    }
    if (node.resolver.resolvesToListLabel(x.getText(), node)) {
        // $ids for ids+=ID etc...
        chunks.add(new ListLabelRef(nodeContext, x.getText()));
        return;
    }
    Rule r = factory.getGrammar().getRule(x.getText());
    if (r != null) {
        // $r for r rule ref
        chunks.add(new LabelRef(nodeContext, getRuleLabel(x.getText())));
    }
}

Example 55 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ActionTranslator method translateAction.

public static List<ActionChunk> translateAction(OutputModelFactory factory, RuleFunction rf, Token tokenWithinAction, ActionAST node) {
    String action = tokenWithinAction.getText();
    if (action != null && action.length() > 0 && action.charAt(0) == '{') {
        int firstCurly = action.indexOf('{');
        int lastCurly = action.lastIndexOf('}');
        if (firstCurly >= 0 && lastCurly >= 0) {
            // trim {...}
            action = action.substring(firstCurly + 1, lastCurly);
        }
    }
    return translateActionChunk(factory, rf, action, node);
}