Examples with BufferedTokenStream org.antlr.v4.runtime.BufferedTokenStream used on opensource projects

Example 11 with BufferedTokenStream

use of org.antlr.v4.runtime.BufferedTokenStream in project antlr4 by antlr.

the class TestTokenStream method testBufferedTokenStreamReuseAfterFill.

/**
 * This is a targeted regression test for antlr/antlr4#1584 ({@link BufferedTokenStream} cannot be reused after EOF).
 */
@Test
public void testBufferedTokenStreamReuseAfterFill() {
    CharStream firstInput = new ANTLRInputStream("A");
    BufferedTokenStream tokenStream = new BufferedTokenStream(new VisitorBasicLexer(firstInput));
    tokenStream.fill();
    Assert.assertEquals(2, tokenStream.size());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
    Assert.assertEquals(Token.EOF, tokenStream.get(1).getType());
    CharStream secondInput = new ANTLRInputStream("AA");
    tokenStream.setTokenSource(new VisitorBasicLexer(secondInput));
    tokenStream.fill();
    Assert.assertEquals(3, tokenStream.size());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(1).getType());
    Assert.assertEquals(Token.EOF, tokenStream.get(2).getType());
}