Examples with TokenStream org.antlr.v4.runtime.TokenStream used on opensource projects

Example 26 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testLookahead.

@Test
public void testLookahead() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = new UnbufferedTokenStream<Token>(lexEngine);
    assertEquals("x", tokens.LT(1).getText());
    assertEquals(" ", tokens.LT(2).getText());
    assertEquals("=", tokens.LT(3).getText());
    assertEquals(" ", tokens.LT(4).getText());
    assertEquals("302", tokens.LT(5).getText());
    assertEquals(";", tokens.LT(6).getText());
}

Example 27 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project presto by prestodb.

the class TypeCalculation method parseTypeCalculation.

private static ParserRuleContext parseTypeCalculation(String calculation) {
    TypeCalculationLexer lexer = new TypeCalculationLexer(new CaseInsensitiveStream(new ANTLRInputStream(calculation)));
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    TypeCalculationParser parser = new TypeCalculationParser(tokenStream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(ERROR_LISTENER);
    parser.removeErrorListeners();
    parser.addErrorListener(ERROR_LISTENER);
    ParserRuleContext tree;
    try {
        // first, try parsing with potentially faster SLL mode
        parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
        tree = parser.typeCalculation();
    } catch (ParseCancellationException ex) {
        // if we fail, parse with LL mode
        // rewind input stream
        tokenStream.reset();
        parser.reset();
        parser.getInterpreter().setPredictionMode(PredictionMode.LL);
        tree = parser.typeCalculation();
    }
    return tree;
}

Example 28 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project sling by apache.

the class ExpressionParser method createParser.

private SightlyParser createParser(String string) {
    SightlyLexer lexer = new SightlyLexer(new ANTLRInputStream(string));
    lexer.removeErrorListeners();
    lexer.addErrorListener(new SightlyParserErrorListener());
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    SightlyParser parser = new SightlyParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(new SightlyParserErrorListener());
    return parser;
}