Examples with LexerInterpreter org.antlr.v4.runtime.LexerInterpreter used on opensource projects

Example 31 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testMarkThenRelease.

@Test
public void testMarkThenRelease() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302 + 1;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    int m = tokens.mark();
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // consume x
    tokens.consume();
    assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
    // ' '
    tokens.consume();
    // =
    tokens.consume();
    // ' '
    tokens.consume();
    assertEquals("302", tokens.LT(1).getText());
    // "x = 302" is in buffer. will kill buffer
    tokens.release(m);
    // 302
    tokens.consume();
    // ' '
    tokens.consume();
    // mark at the +
    m = tokens.mark();
    assertEquals("+", tokens.LT(1).getText());
    // '+'
    tokens.consume();
    // ' '
    tokens.consume();
    // 1
    tokens.consume();
    // ;
    tokens.consume();
    assertEquals("<EOF>", tokens.LT(1).getText());
    // we marked at the +, so that should be the start of the buffer
    assertEquals("[[@6,8:8='+',<5>,1:8], [@7,9:9=' ',<7>,1:9]," + " [@8,10:10='1',<2>,1:10], [@9,11:11=';',<3>,1:11]," + " [@10,12:11='<EOF>',<-1>,1:12]]", tokens.getBuffer().toString());
    tokens.release(m);
}

Example 32 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testNoBuffering.

@Test
public void testNoBuffering() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // move to WS
    tokens.consume();
    assertEquals(" ", tokens.LT(1).getText());
    assertEquals("[[@1,1:1=' ',<7>,1:1]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("[[@2,2:2='=',<4>,1:2]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals(" ", tokens.LT(1).getText());
    assertEquals("[[@3,3:3=' ',<7>,1:3]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals("302", tokens.LT(1).getText());
    assertEquals("[[@4,4:6='302',<2>,1:4]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("[[@5,7:7=';',<3>,1:7]]", tokens.getRemainingBuffer().toString());
}

Example 33 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestUnicodeGrammar method binaryGrammar.

@Test
public void binaryGrammar() throws Exception {
    String grammarText = "grammar Binary;\n" + "r : HEADER PACKET+ FOOTER;\n" + "HEADER : '\\u0002\\u0000\\u0001\\u0007';\n" + "PACKET : '\\u00D0' ('\\u00D1' | '\\u00D2' | '\\u00D3') +;\n" + "FOOTER : '\\u00FF';\n";
    byte[] toParse = new byte[] { (byte) 0x02, (byte) 0x00, (byte) 0x01, (byte) 0x07, (byte) 0xD0, (byte) 0xD2, (byte) 0xD2, (byte) 0xD3, (byte) 0xD3, (byte) 0xD3, (byte) 0xD0, (byte) 0xD3, (byte) 0xD3, (byte) 0xD1, (byte) 0xFF };
    CharStream charStream;
    try (ByteArrayInputStream is = new ByteArrayInputStream(toParse);
        // U+0000 to U+00FF.
        InputStreamReader isr = new InputStreamReader(is, StandardCharsets.ISO_8859_1)) {
        charStream = new ANTLRInputStream(isr);
    }
    Grammar grammar = new Grammar(grammarText);
    LexerInterpreter lexEngine = grammar.createLexerInterpreter(charStream);
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens);
    ParseTree parseTree = parser.parse(grammar.rules.get("r").index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(grammar.getRuleNames());
    String result = Trees.toStringTree(parseTree, nodeTextProvider);
    assertEquals("(r:1 
 ÐÒÒÓÓÓ ÐÓÓÑ ÿ)", result);
}

Example 34 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestParserProfiler method interpAndGetDecisionInfo.

public DecisionInfo[] interpAndGetDecisionInfo(LexerGrammar lg, Grammar g, String startRule, String... input) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(null);
    ParserInterpreter parser = g.createParserInterpreter(null);
    parser.setProfile(true);
    for (String s : input) {
        lexEngine.reset();
        parser.reset();
        lexEngine.setInputStream(new ANTLRInputStream(s));
        CommonTokenStream tokens = new CommonTokenStream(lexEngine);
        parser.setInputStream(tokens);
        Rule r = g.rules.get(startRule);
        if (r == null) {
            return parser.getParseInfo().getDecisionInfo();
        }
        ParserRuleContext t = parser.parse(r.index);
    //			try {
    //				Utils.waitForClose(t.inspect(parser).get());
    //			}
    //			catch (Exception e) {
    //				e.printStackTrace();
    //			}
    //
    //			System.out.println(t.toStringTree(parser));
    }
    return parser.getParseInfo().getDecisionInfo();
}

Example 35 with LexerInterpreter

use of org.antlr.v4.runtime.LexerInterpreter in project antlr4 by antlr.

the class TestUnbufferedCharStream method testAFewTokens.

@Test
public void testAFewTokens() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    TestingUnbufferedCharStream input = createStream("x = 302 * 91 + 20234234 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    // copy text into tokens from char stream
    lexEngine.setTokenFactory(new CommonTokenFactory(true));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    String result = tokens.LT(1).getText();
    String expecting = "x";
    assertEquals(expecting, result);
    tokens.fill();
    expecting = "[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1], [@2,2:2='=',<4>,1:2]," + " [@3,3:3=' ',<7>,1:3], [@4,4:6='302',<2>,1:4], [@5,7:7=' ',<7>,1:7]," + " [@6,8:8='*',<6>,1:8], [@7,9:9=' ',<7>,1:9], [@8,10:11='91',<2>,1:10]," + " [@9,12:12=' ',<7>,1:12], [@10,13:13='+',<5>,1:13], [@11,14:14=' ',<7>,1:14]," + " [@12,15:22='20234234',<2>,1:15], [@13,23:23=' ',<7>,1:23]," + " [@14,24:24='*',<6>,1:24], [@15,25:25=' ',<7>,1:25], [@16,26:26='0',<2>,1:26]," + " [@17,27:27=';',<3>,1:27], [@18,28:27='',<-1>,1:28]]";
    assertEquals(expecting, tokens.getTokens().toString());
}