Examples with ANTLRInputStream org.antlr.v4.runtime.ANTLRInputStream used on opensource projects

Example 6 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testLookback.

@Test
public void testLookback() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    // get x into buffer
    tokens.consume();
    Token t = tokens.LT(-1);
    assertEquals("x", t.getText());
    tokens.consume();
    // consume '='
    tokens.consume();
    t = tokens.LT(-3);
    assertEquals("x", t.getText());
    t = tokens.LT(-2);
    assertEquals(" ", t.getText());
    t = tokens.LT(-1);
    assertEquals("=", t.getText());
}

Example 7 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testFirstToken.

@Test
public void testFirstToken() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    String result = tokens.LT(1).getText();
    String expecting = "x";
    assertEquals(expecting, result);
}

Example 8 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project antlr4 by antlr.

the class TestAmbigParseTrees method testAmbiguousTrees.

public void testAmbiguousTrees(LexerGrammar lg, Grammar g, String startRule, String input, int decision, String expectedAmbigAlts, String overallTree, String[] expectedParseTrees) {
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    final GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    parser.setProfile(true);
    parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
    // PARSE
    int ruleIndex = g.rules.get(startRule).index;
    ParserRuleContext parseTree = parser.parse(ruleIndex);
    assertEquals(overallTree, Trees.toStringTree(parseTree, nodeTextProvider));
    System.out.println();
    DecisionInfo[] decisionInfo = parser.getParseInfo().getDecisionInfo();
    List<AmbiguityInfo> ambiguities = decisionInfo[decision].ambiguities;
    assertEquals(1, ambiguities.size());
    AmbiguityInfo ambiguityInfo = ambiguities.get(0);
    List<ParserRuleContext> ambiguousParseTrees = GrammarParserInterpreter.getAllPossibleParseTrees(g, parser, tokens, decision, ambiguityInfo.ambigAlts, ambiguityInfo.startIndex, ambiguityInfo.stopIndex, ruleIndex);
    assertEquals(expectedAmbigAlts, ambiguityInfo.ambigAlts.toString());
    assertEquals(ambiguityInfo.ambigAlts.cardinality(), ambiguousParseTrees.size());
    for (int i = 0; i < ambiguousParseTrees.size(); i++) {
        ParserRuleContext t = ambiguousParseTrees.get(i);
        assertEquals(expectedParseTrees[i], Trees.toStringTree(t, nodeTextProvider));
    }
}

Example 9 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project antlr4 by antlr.

the class TestLookaheadTrees method testLookaheadTrees.

public void testLookaheadTrees(LexerGrammar lg, Grammar g, String input, String startRuleName, int decision, String[] expectedTrees) {
    int startRuleIndex = g.getRule(startRuleName).index;
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    parser.setProfile(true);
    ParseTree t = parser.parse(startRuleIndex);
    DecisionInfo decisionInfo = parser.getParseInfo().getDecisionInfo()[decision];
    LookaheadEventInfo lookaheadEventInfo = decisionInfo.SLL_MaxLookEvent;
    List<ParserRuleContext> lookaheadParseTrees = GrammarParserInterpreter.getLookaheadParseTrees(g, parser, tokens, startRuleIndex, lookaheadEventInfo.decision, lookaheadEventInfo.startIndex, lookaheadEventInfo.stopIndex);
    assertEquals(expectedTrees.length, lookaheadParseTrees.size());
    for (int i = 0; i < lookaheadParseTrees.size(); i++) {
        ParserRuleContext lt = lookaheadParseTrees.get(i);
        assertEquals(expectedTrees[i], Trees.toStringTree(lt, nodeTextProvider));
    }
}

Example 10 with ANTLRInputStream

use of org.antlr.v4.runtime.ANTLRInputStream in project antlr4 by antlr.

the class ParseTreePatternMatcher method tokenize.

public List<? extends Token> tokenize(String pattern) {
    // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
    List<Chunk> chunks = split(pattern);
    // create token stream from text and tags
    List<Token> tokens = new ArrayList<Token>();
    for (Chunk chunk : chunks) {
        if (chunk instanceof TagChunk) {
            TagChunk tagChunk = (TagChunk) chunk;
            // add special rule token or conjure up new token from name
            if (Character.isUpperCase(tagChunk.getTag().charAt(0))) {
                Integer ttype = parser.getTokenType(tagChunk.getTag());
                if (ttype == Token.INVALID_TYPE) {
                    throw new IllegalArgumentException("Unknown token " + tagChunk.getTag() + " in pattern: " + pattern);
                }
                TokenTagToken t = new TokenTagToken(tagChunk.getTag(), ttype, tagChunk.getLabel());
                tokens.add(t);
            } else if (Character.isLowerCase(tagChunk.getTag().charAt(0))) {
                int ruleIndex = parser.getRuleIndex(tagChunk.getTag());
                if (ruleIndex == -1) {
                    throw new IllegalArgumentException("Unknown rule " + tagChunk.getTag() + " in pattern: " + pattern);
                }
                int ruleImaginaryTokenType = parser.getATNWithBypassAlts().ruleToTokenType[ruleIndex];
                tokens.add(new RuleTagToken(tagChunk.getTag(), ruleImaginaryTokenType, tagChunk.getLabel()));
            } else {
                throw new IllegalArgumentException("invalid tag: " + tagChunk.getTag() + " in pattern: " + pattern);
            }
        } else {
            TextChunk textChunk = (TextChunk) chunk;
            ANTLRInputStream in = new ANTLRInputStream(textChunk.getText());
            lexer.setInputStream(in);
            Token t = lexer.nextToken();
            while (t.getType() != Token.EOF) {
                tokens.add(t);
                t = lexer.nextToken();
            }
        }
    }
    //		System.out.println("tokens="+tokens);
    return tokens;
}