Examples with Token org.antlr.v4.runtime.Token used on opensource projects

Example 91 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestBufferedTokenStream method testCompleteBuffer.

@Test
public void testCompleteBuffer() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    int i = 1;
    Token t = tokens.LT(i);
    while (t.getType() != Token.EOF) {
        i++;
        t = tokens.LT(i);
    }
    // push it past end
    tokens.LT(i++);
    tokens.LT(i++);
    String result = tokens.getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);
}

Example 92 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestCompositeGrammars method testDelegatesSeeSameTokenType.

@Test
public void testDelegatesSeeSameTokenType() throws Exception {
    String slaveS = "parser grammar S;\n" + "tokens { A, B, C }\n" + "x : A ;\n";
    String slaveT = "parser grammar T;\n" + "tokens { C, B, A } // reverse order\n" + "y : A ;\n";
    BaseRuntimeTest.mkdir(tmpdir);
    writeFile(tmpdir, "S.g4", slaveS);
    writeFile(tmpdir, "T.g4", slaveT);
    String master = "// The lexer will create rules to match letters a, b, c.\n" + "// The associated token types A, B, C must have the same value\n" + "// and all import'd parsers.  Since ANTLR regenerates all imports\n" + "// for use with the delegator M, it can generate the same token type\n" + "// mapping in each parser:\n" + "// public static final int C=6;\n" + "// public static final int EOF=-1;\n" + "// public static final int B=5;\n" + "// public static final int WS=7;\n" + "// public static final int A=4;\n" + "grammar M;\n" + "import S,T;\n" + "s : x y ; // matches AA, which should be 'aa'\n" + "B : 'b' ; // another order: B, A, C\n" + "A : 'a' ;\n" + "C : 'c' ;\n" + "WS : (' '|'\\n') -> skip ;\n";
    writeFile(tmpdir, "M.g4", master);
    ErrorQueue equeue = new ErrorQueue();
    Grammar g = new Grammar(tmpdir + "/M.g4", master, equeue);
    String expectedTokenIDToTypeMap = "{EOF=-1, B=1, A=2, C=3, WS=4}";
    String expectedStringLiteralToTypeMap = "{'a'=2, 'b'=1, 'c'=3}";
    String expectedTypeToTokenList = "[B, A, C, WS]";
    assertEquals(expectedTokenIDToTypeMap, g.tokenNameToTypeMap.toString());
    assertEquals(expectedStringLiteralToTypeMap, sort(g.stringLiteralToTypeMap).toString());
    assertEquals(expectedTypeToTokenList, realElements(g.typeToTokenList).toString());
    assertEquals("unexpected errors: " + equeue, 0, equeue.errors.size());
}

Example 93 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 94 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}

Example 95 with Token

use of org.antlr.v4.runtime.Token in project antlr4 by antlr.

the class ActionTranslator method getRulePropertyRef.

RulePropertyRef getRulePropertyRef(Token x, Token prop) {
    Grammar g = factory.getGrammar();
    try {
        Class<? extends RulePropertyRef> c = rulePropToModelMap.get(prop.getText());
        Constructor<? extends RulePropertyRef> ctor = c.getConstructor(StructDecl.class, String.class);
        RulePropertyRef ref = ctor.newInstance(nodeContext, getRuleLabel(x.getText()));
        return ref;
    } catch (Exception e) {
        g.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e, prop.getText());
    }
    return null;
}