Examples with IntervalSet org.antlr.v4.runtime.misc.IntervalSet used on opensource projects

Example 76 with IntervalSet

use of org.antlr.v4.runtime.misc.IntervalSet in project antlr4 by antlr.

the class TestTokenPositionOptions method testLeftRecursionRewrite.

@Test
public void testLeftRecursionRewrite() throws Exception {
    Grammar g = new Grammar("grammar T;\n" + "s : e ';' ;\n" + "e : e '*' e\n" + "  | e '+' e\n" + "  | e '.' ID\n" + "  | '-' e\n" + "  | ID\n" + "  ;\n" + "ID : [a-z]+ ;\n");
    String expectedTree = "(COMBINED_GRAMMAR T (RULES (RULE s (BLOCK (ALT e ';'))) (RULE e (BLOCK (ALT (BLOCK (ALT {} ('-' (ELEMENT_OPTIONS (= tokenIndex 43))) (e (ELEMENT_OPTIONS (= tokenIndex 45) (= p 2)))) (ALT (ID (ELEMENT_OPTIONS (= tokenIndex 49))))) (* (BLOCK (ALT ({precpred(_ctx, 5)}? (ELEMENT_OPTIONS (= p 5))) ('*' (ELEMENT_OPTIONS (= tokenIndex 21))) (e (ELEMENT_OPTIONS (= tokenIndex 23) (= p 6)))) (ALT ({precpred(_ctx, 4)}? (ELEMENT_OPTIONS (= p 4))) ('+' (ELEMENT_OPTIONS (= tokenIndex 29))) (e (ELEMENT_OPTIONS (= tokenIndex 31) (= p 5)))) (ALT ({precpred(_ctx, 3)}? (ELEMENT_OPTIONS (= p 3))) ('.' (ELEMENT_OPTIONS (= tokenIndex 37))) (ID (ELEMENT_OPTIONS (= tokenIndex 39)))))))))))";
    assertEquals(expectedTree, g.ast.toStringTree());
    String expectedElementTokens = "[@5,11:11='s',<57>,2:0]\n" + "[@9,15:15='e',<57>,2:4]\n" + "[@11,17:19='';'',<62>,2:6]\n" + "[@15,23:23='e',<57>,3:0]\n" + "[@43,64:66=''-'',<62>,6:4]\n" + "[@45,68:68='e',<57>,6:8]\n" + "[@49,74:75='ID',<66>,7:4]\n" + "[@21,29:31=''*'',<62>,3:6]\n" + "[@23,33:33='e',<57>,3:10]\n" + "[@29,41:43=''+'',<62>,4:6]\n" + "[@31,45:45='e',<57>,4:10]\n" + "[@37,53:55=''.'',<62>,5:6]\n" + "[@39,57:58='ID',<66>,5:10]";
    IntervalSet types = new IntervalSet(ANTLRParser.TOKEN_REF, ANTLRParser.STRING_LITERAL, ANTLRParser.RULE_REF);
    List<GrammarAST> nodes = g.ast.getNodesWithTypePreorderDFS(types);
    List<Token> tokens = new ArrayList<Token>();
    for (GrammarAST node : nodes) {
        tokens.add(node.getToken());
    }
    assertEquals(expectedElementTokens, Utils.join(tokens.toArray(), "\n"));
}

Example 77 with IntervalSet

use of org.antlr.v4.runtime.misc.IntervalSet in project antlr4 by antlr.

the class TestTokenPositionOptions method testLeftRecursionWithLabels.

@Test
public void testLeftRecursionWithLabels() throws Exception {
    Grammar g = new Grammar("grammar T;\n" + "s : e ';' ;\n" + "e : e '*' x=e\n" + "  | e '+' e\n" + "  | e '.' y=ID\n" + "  | '-' e\n" + "  | ID\n" + "  ;\n" + "ID : [a-z]+ ;\n");
    String expectedTree = "(COMBINED_GRAMMAR T (RULES (RULE s (BLOCK (ALT e ';'))) (RULE e (BLOCK (ALT (BLOCK (ALT {} ('-' (ELEMENT_OPTIONS (= tokenIndex 47))) (e (ELEMENT_OPTIONS (= tokenIndex 49) (= p 2)))) (ALT (ID (ELEMENT_OPTIONS (= tokenIndex 53))))) (* (BLOCK (ALT ({precpred(_ctx, 5)}? (ELEMENT_OPTIONS (= p 5))) ('*' (ELEMENT_OPTIONS (= tokenIndex 21))) (= x (e (ELEMENT_OPTIONS (= tokenIndex 25) (= p 6))))) (ALT ({precpred(_ctx, 4)}? (ELEMENT_OPTIONS (= p 4))) ('+' (ELEMENT_OPTIONS (= tokenIndex 31))) (e (ELEMENT_OPTIONS (= tokenIndex 33) (= p 5)))) (ALT ({precpred(_ctx, 3)}? (ELEMENT_OPTIONS (= p 3))) ('.' (ELEMENT_OPTIONS (= tokenIndex 39))) (= y (ID (ELEMENT_OPTIONS (= tokenIndex 43))))))))))))";
    assertEquals(expectedTree, g.ast.toStringTree());
    String expectedElementTokens = "[@5,11:11='s',<57>,2:0]\n" + "[@9,15:15='e',<57>,2:4]\n" + "[@11,17:19='';'',<62>,2:6]\n" + "[@15,23:23='e',<57>,3:0]\n" + "[@47,68:70=''-'',<62>,6:4]\n" + "[@49,72:72='e',<57>,6:8]\n" + "[@53,78:79='ID',<66>,7:4]\n" + "[@21,29:31=''*'',<62>,3:6]\n" + "[@25,35:35='e',<57>,3:12]\n" + "[@31,43:45=''+'',<62>,4:6]\n" + "[@33,47:47='e',<57>,4:10]\n" + "[@39,55:57=''.'',<62>,5:6]\n" + "[@43,61:62='ID',<66>,5:12]";
    IntervalSet types = new IntervalSet(ANTLRParser.TOKEN_REF, ANTLRParser.STRING_LITERAL, ANTLRParser.RULE_REF);
    List<GrammarAST> nodes = g.ast.getNodesWithTypePreorderDFS(types);
    List<Token> tokens = new ArrayList<Token>();
    for (GrammarAST node : nodes) {
        tokens.add(node.getToken());
    }
    assertEquals(expectedElementTokens, Utils.join(tokens.toArray(), "\n"));
}

Example 78 with IntervalSet

use of org.antlr.v4.runtime.misc.IntervalSet in project antlr4 by antlr.

the class LeftRecursiveRuleAnalyzer method text.

public String text(GrammarAST t) {
    if (t == null)
        return "";
    int tokenStartIndex = t.getTokenStartIndex();
    int tokenStopIndex = t.getTokenStopIndex();
    // ignore tokens from existing option subtrees like:
    //    (ELEMENT_OPTIONS (= assoc right))
    //
    // element options are added back according to the values in the map
    // returned by getOptions().
    IntervalSet ignore = new IntervalSet();
    List<GrammarAST> optionsSubTrees = t.getNodesWithType(ELEMENT_OPTIONS);
    for (GrammarAST sub : optionsSubTrees) {
        ignore.add(sub.getTokenStartIndex(), sub.getTokenStopIndex());
    }
    // Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
    // but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
    // and add the tokenIndex option when writing these tokens.
    IntervalSet noOptions = new IntervalSet();
    List<GrammarAST> labeledSubTrees = t.getNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
    for (GrammarAST sub : labeledSubTrees) {
        noOptions.add(sub.getChild(0).getTokenStartIndex());
    }
    StringBuilder buf = new StringBuilder();
    int i = tokenStartIndex;
    while (i <= tokenStopIndex) {
        if (ignore.contains(i)) {
            i++;
            continue;
        }
        Token tok = tokenStream.get(i);
        // Compute/hold any element options
        StringBuilder elementOptions = new StringBuilder();
        if (!noOptions.contains(i)) {
            GrammarAST node = t.getNodeWithTokenIndex(tok.getTokenIndex());
            if (node != null && (tok.getType() == TOKEN_REF || tok.getType() == STRING_LITERAL || tok.getType() == RULE_REF)) {
                elementOptions.append("tokenIndex=").append(tok.getTokenIndex());
            }
            if (node instanceof GrammarASTWithOptions) {
                GrammarASTWithOptions o = (GrammarASTWithOptions) node;
                for (Map.Entry<String, GrammarAST> entry : o.getOptions().entrySet()) {
                    if (elementOptions.length() > 0) {
                        elementOptions.append(',');
                    }
                    elementOptions.append(entry.getKey());
                    elementOptions.append('=');
                    elementOptions.append(entry.getValue().getText());
                }
            }
        }
        // add actual text of the current token to the rewritten alternative
        buf.append(tok.getText());
        // move to the next token
        i++;
        // Are there args on a rule?
        if (tok.getType() == RULE_REF && i <= tokenStopIndex && tokenStream.get(i).getType() == ARG_ACTION) {
            buf.append('[' + tokenStream.get(i).getText() + ']');
            i++;
        }
        // now that we have the actual element, we can add the options.
        if (elementOptions.length() > 0) {
            buf.append('<').append(elementOptions).append('>');
        }
    }
    return buf.toString();
}

Example 79 with IntervalSet

use of org.antlr.v4.runtime.misc.IntervalSet in project antlr4 by antlr.

the class LexerATNFactory method set.

@Override
public Handle set(GrammarAST associatedAST, List<GrammarAST> alts, boolean invert) {
    ATNState left = newState(associatedAST);
    ATNState right = newState(associatedAST);
    IntervalSet set = new IntervalSet();
    for (GrammarAST t : alts) {
        if (t.getType() == ANTLRParser.RANGE) {
            int a = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(0).getText());
            int b = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(1).getText());
            if (checkRange((GrammarAST) t.getChild(0), (GrammarAST) t.getChild(1), a, b)) {
                checkSetCollision(associatedAST, set, a, b);
                set.add(a, b);
            }
        } else if (t.getType() == ANTLRParser.LEXER_CHAR_SET) {
            set.addAll(getSetFromCharSetLiteral(t));
        } else if (t.getType() == ANTLRParser.STRING_LITERAL) {
            int c = CharSupport.getCharValueFromGrammarCharLiteral(t.getText());
            if (c != -1) {
                checkSetCollision(associatedAST, set, c);
                set.add(c);
            } else {
                g.tool.errMgr.grammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET, g.fileName, t.getToken(), t.getText());
            }
        } else if (t.getType() == ANTLRParser.TOKEN_REF) {
            g.tool.errMgr.grammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET, g.fileName, t.getToken(), t.getText());
        }
    }
    if (invert) {
        left.addTransition(new NotSetTransition(right, set));
    } else {
        Transition transition;
        if (set.getIntervals().size() == 1) {
            Interval interval = set.getIntervals().get(0);
            transition = CodePointTransitions.createWithCodePointRange(right, interval.a, interval.b);
        } else {
            transition = new SetTransition(right, set);
        }
        left.addTransition(transition);
    }
    associatedAST.atnState = left;
    return new Handle(left, right);
}

Example 80 with IntervalSet

use of org.antlr.v4.runtime.misc.IntervalSet in project antlr4 by antlr.

the class AnalysisPipeline method processParser.

protected void processParser() {
    g.decisionLOOK = new ArrayList<IntervalSet[]>(g.atn.getNumberOfDecisions() + 1);
    for (DecisionState s : g.atn.decisionToState) {
        g.tool.log("LL1", "\nDECISION " + s.decision + " in rule " + g.getRule(s.ruleIndex).name);
        IntervalSet[] look;
        if (s.nonGreedy) {
            // nongreedy decisions can't be LL(1)
            look = new IntervalSet[s.getNumberOfTransitions() + 1];
        } else {
            LL1Analyzer anal = new LL1Analyzer(g.atn);
            look = anal.getDecisionLookahead(s);
            g.tool.log("LL1", "look=" + Arrays.toString(look));
        }
        assert s.decision + 1 >= g.decisionLOOK.size();
        Utils.setSize(g.decisionLOOK, s.decision + 1);
        g.decisionLOOK.set(s.decision, look);
        g.tool.log("LL1", "LL(1)? " + disjoint(look));
    }
}