Examples with EOF org.antlr.v4.runtime.Recognizer.EOF used on opensource projects

Example 86 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestTokenTypeAssignment method testPredDoesNotHideNameToLiteralMapInLexer.

@Test
public void testPredDoesNotHideNameToLiteralMapInLexer() throws Exception {
    // 'x' is token and char in lexer rule
    Grammar g = new Grammar("grammar t;\n" + "a : 'x' X ; \n" + // must match as alias even with pred
    "X: 'x' {true}?;\n");
    assertEquals("{'x'=1}", g.stringLiteralToTypeMap.toString());
    assertEquals("{EOF=-1, X=1}", g.tokenNameToTypeMap.toString());
    // pushed in lexer from parser
    assertEquals("{'x'=1}", g.implicitLexer.stringLiteralToTypeMap.toString());
    assertEquals("{EOF=-1, X=1}", g.implicitLexer.tokenNameToTypeMap.toString());
}

Example 87 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestUnbufferedTokenStream method testMarkThenRelease.

@Test
public void testMarkThenRelease() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = new ANTLRInputStream(new StringReader("x = 302 + 1;"));
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TestingUnbufferedTokenStream<Token> tokens = new TestingUnbufferedTokenStream<Token>(lexEngine);
    int m = tokens.mark();
    assertEquals("[[@0,0:0='x',<1>,1:0]]", tokens.getBuffer().toString());
    assertEquals("x", tokens.LT(1).getText());
    // consume x
    tokens.consume();
    assertEquals("[[@0,0:0='x',<1>,1:0], [@1,1:1=' ',<7>,1:1]]", tokens.getBuffer().toString());
    // ' '
    tokens.consume();
    // =
    tokens.consume();
    // ' '
    tokens.consume();
    assertEquals("302", tokens.LT(1).getText());
    // "x = 302" is in buffer. will kill buffer
    tokens.release(m);
    // 302
    tokens.consume();
    // ' '
    tokens.consume();
    // mark at the +
    m = tokens.mark();
    assertEquals("+", tokens.LT(1).getText());
    // '+'
    tokens.consume();
    // ' '
    tokens.consume();
    // 1
    tokens.consume();
    // ;
    tokens.consume();
    assertEquals("<EOF>", tokens.LT(1).getText());
    // we marked at the +, so that should be the start of the buffer
    assertEquals("[[@6,8:8='+',<5>,1:8], [@7,9:9=' ',<7>,1:9]," + " [@8,10:10='1',<2>,1:10], [@9,11:11=';',<3>,1:11]," + " [@10,12:11='<EOF>',<-1>,1:12]]", tokens.getBuffer().toString());
    tokens.release(m);
}

Example 88 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestVisitors method testCalculatorVisitor.

/**
 * This test verifies that the visitor correctly dispatches calls for labeled outer alternatives.
 */
@Test
public void testCalculatorVisitor() {
    String input = "2 + 8 / 2";
    VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input));
    VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer));
    VisitorCalcParser.SContext context = parser.s();
    Assert.assertEquals("(s (expr (expr 2) + (expr (expr 8) / (expr 2))) <EOF>)", context.toStringTree(parser));
    VisitorCalcVisitor<Integer> listener = new VisitorCalcBaseVisitor<Integer>() {

        @Override
        public Integer visitS(VisitorCalcParser.SContext ctx) {
            return visit(ctx.expr());
        }

        @Override
        public Integer visitNumber(VisitorCalcParser.NumberContext ctx) {
            return Integer.valueOf(ctx.INT().getText());
        }

        @Override
        public Integer visitMultiply(VisitorCalcParser.MultiplyContext ctx) {
            Integer left = visit(ctx.expr(0));
            Integer right = visit(ctx.expr(1));
            if (ctx.MUL() != null) {
                return left * right;
            } else {
                return left / right;
            }
        }

        @Override
        public Integer visitAdd(VisitorCalcParser.AddContext ctx) {
            Integer left = visit(ctx.expr(0));
            Integer right = visit(ctx.expr(1));
            if (ctx.ADD() != null) {
                return left + right;
            } else {
                return left - right;
            }
        }

        @Override
        protected Integer defaultResult() {
            throw new RuntimeException("Should not be reachable");
        }

        @Override
        protected Integer aggregateResult(Integer aggregate, Integer nextResult) {
            throw new RuntimeException("Should not be reachable");
        }
    };
    int result = listener.visit(context);
    int expected = 6;
    Assert.assertEquals(expected, result);
}

Example 89 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestExpectedTokens method testOptionalSubrule.

@Test
public void testOptionalSubrule() throws Exception {
    String gtext = "parser grammar T;\n" + "a : A B? C ;\n";
    Grammar g = new Grammar(gtext);
    String atnText = "RuleStart_a_0->s2\n" + "s2-A->BlockStart_4\n" + "BlockStart_4->s3\n" + "BlockStart_4->BlockEnd_5\n" + "s3-B->BlockEnd_5\n" + "BlockEnd_5->s6\n" + "s6-C->s7\n" + "s7->RuleStop_a_1\n" + "RuleStop_a_1-EOF->s8\n";
    checkRuleATN(g, "a", atnText);
    ATN atn = g.getATN();
    int blkStartStateNumber = 4;
    IntervalSet tokens = atn.getExpectedTokens(blkStartStateNumber, null);
    assertEquals("{B, C}", tokens.toString(g.getTokenNames()));
}

Example 90 with EOF

use of org.antlr.v4.runtime.Recognizer.EOF in project antlr4 by antlr.

the class TestExpectedTokens method testFollowIncluded.

@Test
public void testFollowIncluded() throws Exception {
    String gtext = "parser grammar T;\n" + "a : b A ;\n" + "b : B | ;";
    Grammar g = new Grammar(gtext);
    String atnText = "RuleStart_a_0->s4\n" + "s4-b->RuleStart_b_2\n" + "s5-A->s6\n" + "s6->RuleStop_a_1\n" + "RuleStop_a_1-EOF->s11\n";
    checkRuleATN(g, "a", atnText);
    atnText = "RuleStart_b_2->BlockStart_9\n" + "BlockStart_9->s7\n" + "BlockStart_9->s8\n" + "s7-B->BlockEnd_10\n" + "s8->BlockEnd_10\n" + "BlockEnd_10->RuleStop_b_3\n" + "RuleStop_b_3->s5\n";
    checkRuleATN(g, "b", atnText);
    ATN atn = g.getATN();
    // From the start of 'b' with empty stack, can only see B and EOF
    int blkStartStateNumber = 9;
    IntervalSet tokens = atn.getExpectedTokens(blkStartStateNumber, RuleContext.EMPTY);
    assertEquals("{<EOF>, B}", tokens.toString(g.getTokenNames()));
    // Now call from 'a'
    tokens = atn.getExpectedTokens(blkStartStateNumber, new ParserRuleContext(ParserRuleContext.EMPTY, 4));
    assertEquals("{A, B}", tokens.toString(g.getTokenNames()));
}