Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 86 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LeftRecursiveRuleAnalyzer method stripLeftRecursion.

// TODO: this strips the tree properly, but since text()
// uses the start of stop token index and gets text from that
// ineffectively ignores this routine.
public GrammarAST stripLeftRecursion(GrammarAST altAST) {
    GrammarAST lrlabel = null;
    GrammarAST first = (GrammarAST) altAST.getChild(0);
    int leftRecurRuleIndex = 0;
    if (first.getType() == ELEMENT_OPTIONS) {
        first = (GrammarAST) altAST.getChild(1);
        leftRecurRuleIndex = 1;
    }
    // if label=rule
    Tree rref = first.getChild(1);
    if ((first.getType() == RULE_REF && first.getText().equals(ruleName)) || (rref != null && rref.getType() == RULE_REF && rref.getText().equals(ruleName))) {
        if (first.getType() == ASSIGN || first.getType() == PLUS_ASSIGN)
            lrlabel = (GrammarAST) first.getChild(0);
        // remove rule ref (first child unless options present)
        altAST.deleteChild(leftRecurRuleIndex);
        // reset index so it prints properly (sets token range of
        // ALT to start to right of left recur rule we deleted)
        GrammarAST newFirstChild = (GrammarAST) altAST.getChild(leftRecurRuleIndex);
        altAST.setTokenStartIndex(newFirstChild.getTokenStartIndex());
    }
    return lrlabel;
}

Example 87 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LeftRecursiveRuleAnalyzer method text.

public String text(GrammarAST t) {
    if (t == null)
        return "";
    int tokenStartIndex = t.getTokenStartIndex();
    int tokenStopIndex = t.getTokenStopIndex();
    // ignore tokens from existing option subtrees like:
    //    (ELEMENT_OPTIONS (= assoc right))
    //
    // element options are added back according to the values in the map
    // returned by getOptions().
    IntervalSet ignore = new IntervalSet();
    List<GrammarAST> optionsSubTrees = t.getNodesWithType(ELEMENT_OPTIONS);
    for (GrammarAST sub : optionsSubTrees) {
        ignore.add(sub.getTokenStartIndex(), sub.getTokenStopIndex());
    }
    // Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
    // but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
    // and add the tokenIndex option when writing these tokens.
    IntervalSet noOptions = new IntervalSet();
    List<GrammarAST> labeledSubTrees = t.getNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
    for (GrammarAST sub : labeledSubTrees) {
        noOptions.add(sub.getChild(0).getTokenStartIndex());
    }
    StringBuilder buf = new StringBuilder();
    int i = tokenStartIndex;
    while (i <= tokenStopIndex) {
        if (ignore.contains(i)) {
            i++;
            continue;
        }
        Token tok = tokenStream.get(i);
        // Compute/hold any element options
        StringBuilder elementOptions = new StringBuilder();
        if (!noOptions.contains(i)) {
            GrammarAST node = t.getNodeWithTokenIndex(tok.getTokenIndex());
            if (node != null && (tok.getType() == TOKEN_REF || tok.getType() == STRING_LITERAL || tok.getType() == RULE_REF)) {
                elementOptions.append("tokenIndex=").append(tok.getTokenIndex());
            }
            if (node instanceof GrammarASTWithOptions) {
                GrammarASTWithOptions o = (GrammarASTWithOptions) node;
                for (Map.Entry<String, GrammarAST> entry : o.getOptions().entrySet()) {
                    if (elementOptions.length() > 0) {
                        elementOptions.append(',');
                    }
                    elementOptions.append(entry.getKey());
                    elementOptions.append('=');
                    elementOptions.append(entry.getValue().getText());
                }
            }
        }
        // add actual text of the current token to the rewritten alternative
        buf.append(tok.getText());
        // move to the next token
        i++;
        // Are there args on a rule?
        if (tok.getType() == RULE_REF && i <= tokenStopIndex && tokenStream.get(i).getType() == ARG_ACTION) {
            buf.append('[' + tokenStream.get(i).getText() + ']');
            i++;
        }
        // now that we have the actual element, we can add the options.
        if (elementOptions.length() > 0) {
            buf.append('<').append(elementOptions).append('>');
        }
    }
    return buf.toString();
}

Example 88 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LeftRecursiveRuleTransformer method translateLeftRecursiveRules.

public void translateLeftRecursiveRules() {
    String language = g.getOptionString("language");
    // translate all recursive rules
    List<String> leftRecursiveRuleNames = new ArrayList<String>();
    for (Rule r : rules) {
        if (!Grammar.isTokenName(r.name)) {
            if (LeftRecursiveRuleAnalyzer.hasImmediateRecursiveRuleRefs(r.ast, r.name)) {
                boolean fitsPattern = translateLeftRecursiveRule(ast, (LeftRecursiveRule) r, language);
                if (fitsPattern) {
                    leftRecursiveRuleNames.add(r.name);
                } else {
                    // better given an error that non-conforming left-recursion exists
                    tool.errMgr.grammarError(ErrorType.NONCONFORMING_LR_RULE, g.fileName, ((GrammarAST) r.ast.getChild(0)).token, r.name);
                }
            }
        }
    }
    // update all refs to recursive rules to have [0] argument
    for (GrammarAST r : ast.getNodesWithType(ANTLRParser.RULE_REF)) {
        // must be rule def
        if (r.getParent().getType() == ANTLRParser.RULE)
            continue;
        // already has arg; must be in rewritten rule
        if (((GrammarASTWithOptions) r).getOptionString(PRECEDENCE_OPTION_NAME) != null)
            continue;
        if (leftRecursiveRuleNames.contains(r.getText())) {
            // found ref to recursive rule not already rewritten with arg
            ((GrammarASTWithOptions) r).setOption(PRECEDENCE_OPTION_NAME, (GrammarAST) new GrammarASTAdaptor().create(ANTLRParser.INT, "0"));
        }
    }
}

Example 89 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LeftRecursiveRuleTransformer method translateLeftRecursiveRule.

/** Return true if successful */
public boolean translateLeftRecursiveRule(GrammarRootAST ast, LeftRecursiveRule r, String language) {
    //tool.log("grammar", ruleAST.toStringTree());
    GrammarAST prevRuleAST = r.ast;
    String ruleName = prevRuleAST.getChild(0).getText();
    LeftRecursiveRuleAnalyzer leftRecursiveRuleWalker = new LeftRecursiveRuleAnalyzer(prevRuleAST, tool, ruleName, language);
    boolean isLeftRec;
    try {
        //			System.out.println("TESTING ---------------\n"+
        //							   leftRecursiveRuleWalker.text(ruleAST));
        isLeftRec = leftRecursiveRuleWalker.rec_rule();
    } catch (RecognitionException re) {
        // didn't match; oh well
        isLeftRec = false;
    }
    if (!isLeftRec)
        return false;
    // replace old rule's AST; first create text of altered rule
    GrammarAST RULES = (GrammarAST) ast.getFirstChildWithType(ANTLRParser.RULES);
    String newRuleText = leftRecursiveRuleWalker.getArtificialOpPrecRule();
    //		System.out.println("created: "+newRuleText);
    // now parse within the context of the grammar that originally created
    // the AST we are transforming. This could be an imported grammar so
    // we cannot just reference this.g because the role might come from
    // the imported grammar and not the root grammar (this.g)
    RuleAST t = parseArtificialRule(prevRuleAST.g, newRuleText);
    // reuse the name token from the original AST since it refers to the proper source location in the original grammar
    ((GrammarAST) t.getChild(0)).token = ((GrammarAST) prevRuleAST.getChild(0)).getToken();
    // update grammar AST and set rule's AST.
    RULES.setChild(prevRuleAST.getChildIndex(), t);
    r.ast = t;
    // Reduce sets in newly created rule tree
    GrammarTransformPipeline transform = new GrammarTransformPipeline(g, g.tool);
    transform.reduceBlocksToSets(r.ast);
    transform.expandParameterizedLoops(r.ast);
    // Rerun semantic checks on the new rule
    RuleCollector ruleCollector = new RuleCollector(g);
    ruleCollector.visit(t, "rule");
    BasicSemanticChecks basics = new BasicSemanticChecks(g, ruleCollector);
    // disable the assoc element option checks because they are already
    // handled for the pre-transformed rule.
    basics.checkAssocElementOption = false;
    basics.visit(t, "rule");
    // track recursive alt info for codegen
    r.recPrimaryAlts = new ArrayList<LeftRecursiveRuleAltInfo>();
    r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.prefixAndOtherAlts);
    if (r.recPrimaryAlts.isEmpty()) {
        tool.errMgr.grammarError(ErrorType.NO_NON_LR_ALTS, g.fileName, ((GrammarAST) r.ast.getChild(0)).getToken(), r.name);
    }
    r.recOpAlts = new OrderedHashMap<Integer, LeftRecursiveRuleAltInfo>();
    r.recOpAlts.putAll(leftRecursiveRuleWalker.binaryAlts);
    r.recOpAlts.putAll(leftRecursiveRuleWalker.ternaryAlts);
    r.recOpAlts.putAll(leftRecursiveRuleWalker.suffixAlts);
    // walk alt info records and set their altAST to point to appropriate ALT subtree
    // from freshly created AST
    setAltASTPointers(r, t);
    // update Rule to just one alt and add prec alt
    ActionAST arg = (ActionAST) r.ast.getFirstChildWithType(ANTLRParser.ARG_ACTION);
    if (arg != null) {
        r.args = ScopeParser.parseTypedArgList(arg, arg.getText(), g);
        r.args.type = AttributeDict.DictType.ARG;
        r.args.ast = arg;
        // todo: isn't this Rule or something?
        arg.resolver = r.alt[1];
    }
    // these are so $label in action translation works
    for (Pair<GrammarAST, String> pair : leftRecursiveRuleWalker.leftRecursiveRuleRefLabels) {
        GrammarAST labelNode = pair.a;
        GrammarAST labelOpNode = (GrammarAST) labelNode.getParent();
        GrammarAST elementNode = (GrammarAST) labelOpNode.getChild(1);
        LabelElementPair lp = new LabelElementPair(g, labelNode, elementNode, labelOpNode.getType());
        r.alt[1].labelDefs.map(labelNode.getText(), lp);
    }
    // copy to rule from walker
    r.leftRecursiveRuleRefLabels = leftRecursiveRuleWalker.leftRecursiveRuleRefLabels;
    tool.log("grammar", "added: " + t.toStringTree());
    return true;
}

Example 90 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class LexerATNFactory method createATN.

@Override
public ATN createATN() {
    // BUILD ALL START STATES (ONE PER MODE)
    Set<String> modes = ((LexerGrammar) g).modes.keySet();
    for (String modeName : modes) {
        // create s0, start state; implied Tokens rule node
        TokensStartState startState = newState(TokensStartState.class, null);
        atn.modeNameToStartState.put(modeName, startState);
        atn.modeToStartState.add(startState);
        atn.defineDecisionState(startState);
    }
    // INIT ACTION, RULE->TOKEN_TYPE MAP
    atn.ruleToTokenType = new int[g.rules.size()];
    for (Rule r : g.rules.values()) {
        atn.ruleToTokenType[r.index] = g.getTokenType(r.name);
    }
    // CREATE ATN FOR EACH RULE
    _createATN(g.rules.values());
    atn.lexerActions = new LexerAction[indexToActionMap.size()];
    for (Map.Entry<Integer, LexerAction> entry : indexToActionMap.entrySet()) {
        atn.lexerActions[entry.getKey()] = entry.getValue();
    }
    // LINK MODE START STATE TO EACH TOKEN RULE
    for (String modeName : modes) {
        List<Rule> rules = ((LexerGrammar) g).modes.get(modeName);
        TokensStartState startState = atn.modeNameToStartState.get(modeName);
        for (Rule r : rules) {
            if (!r.isFragment()) {
                RuleStartState s = atn.ruleToStartState[r.index];
                epsilon(startState, s);
            }
        }
    }
    ATNOptimizer.optimize(g, atn);
    return atn;
}