Examples with CharStream org.antlr.v4.runtime.CharStream used on opensource projects

Example 91 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testMarkThenRelease.

@Test
public void testMarkThenRelease() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302 + 1;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    int m = tokens.mark();
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // consume x
    tokens.consume();
    assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
    // ' '
    tokens.consume();
    // =
    tokens.consume();
    // ' '
    tokens.consume();
    assertEquals("302", tokens.LT(1).getText());
    // "x = 302" is in buffer. will kill buffer
    tokens.release(m);
    // 302
    tokens.consume();
    // ' '
    tokens.consume();
    // mark at the +
    m = tokens.mark();
    assertEquals("+", tokens.LT(1).getText());
    // '+'
    tokens.consume();
    // ' '
    tokens.consume();
    // 1
    tokens.consume();
    // ;
    tokens.consume();
    assertEquals("<EOF>", tokens.LT(1).getText());
    // we marked at the +, so that should be the start of the buffer
    assertEquals("[[@6,8:8='+',<5>,1:8], [@7,9:9=' ',<7>,1:9]," + " [@8,10:10='1',<2>,1:10], [@9,11:11=';',<3>,1:11]," + " [@10,12:11='<EOF>',<-1>,1:12]]", tokens.getBuffer().toString());
    tokens.release(m);
}

Example 92 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testNoBuffering.

@Test
public void testNoBuffering() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // move to WS
    tokens.consume();
    assertEquals(" ", tokens.LT(1).getText());
    assertEquals("[[@1,1:1=' ',<7>,1:1]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("[[@2,2:2='=',<4>,1:2]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals(" ", tokens.LT(1).getText());
    assertEquals("[[@3,3:3=' ',<7>,1:3]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals("302", tokens.LT(1).getText());
    assertEquals("[[@4,4:6='302',<2>,1:4]]", tokens.getRemainingBuffer().toString());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("[[@5,7:7=';',<3>,1:7]]", tokens.getRemainingBuffer().toString());
}

Example 93 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TestUnbufferedCharStream method testLastChar.

@Test
public void testLastChar() {
    CharStream input = createStream("abcdef");
    input.consume();
    assertEquals('a', input.LA(-1));
    int m1 = input.mark();
    input.consume();
    input.consume();
    input.consume();
    assertEquals('d', input.LA(-1));
    input.seek(2);
    assertEquals('b', input.LA(-1));
    input.release(m1);
    input.seek(3);
    assertEquals('c', input.LA(-1));
    // this special case is not required by the IntStream interface, but
    // UnbufferedCharStream allows it so we have to make sure the resulting
    // state is consistent
    input.seek(2);
    assertEquals('b', input.LA(-1));
}

Example 94 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TestUnbufferedCharStream method testMarkPassedToSeek.

/**
 * It is not valid to pass a mark to {@link IntStream#seek}, but
 * {@link UnbufferedCharStream} creates marks in such a way that this
 * invalid usage results in an {@link IllegalArgumentException}.
 */
@Test(expected = IllegalArgumentException.class)
public void testMarkPassedToSeek() {
    CharStream input = createStream("");
    int m1 = input.mark();
    input.seek(m1);
}

Example 95 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TimeLexerSpeed method lex_legacy_java_utf8.

public void lex_legacy_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    try (InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(Parser_java_file);
        InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
        BufferedReader br = new BufferedReader(isr)) {
        CharStream input = new ANTLRInputStream(br);
        JavaLexer lexer = new JavaLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
    }
}