Examples with CharStream org.antlr.v4.runtime.CharStream used on opensource projects

Example 41 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCharStreams method fromSMPUTF8PathHasExpectedSize.

@Test
public void fromSMPUTF8PathHasExpectedSize() throws Exception {
    File p = folder.newFile();
    Utils.writeFile(p, "hello \uD83C\uDF0E".getBytes(Charset.forName("UTF-8")));
    CharStream s = CharStreams.fromFile(p);
    assertEquals(7, s.size());
    assertEquals(0, s.index());
    assertEquals("hello \uD83C\uDF0E", s.toString());
    assertEquals(p.toString(), s.getSourceName());
}

Example 42 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCharStreams method fromSMPUTF8ChannelHasExpectedSize.

@Test
public void fromSMPUTF8ChannelHasExpectedSize() throws Exception {
    File p = folder.newFile();
    Utils.writeFile(p, "hello \uD83C\uDF0E".getBytes(Charset.forName("UTF-8")));
    ReadableByteChannel c = Channels.newChannel(new FileInputStream(p));
    try {
        CharStream s = CharStreams.fromChannel(c, 4096, CodingErrorAction.REPLACE, "foo");
        assertEquals(7, s.size());
        assertEquals(0, s.index());
        assertEquals("hello \uD83C\uDF0E", s.toString());
        assertEquals("foo", s.getSourceName());
    } finally {
        c.close();
    }
}

Example 43 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public TokenFactory getTokenFactory() {
            return CommonTokenFactory.DEFAULT;
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}

Example 44 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testSingleEOF.

@Test
public void testSingleEOF() throws Exception {
    TokenSource lexer = new TokenSource() {

        @Override
        public Token nextToken() {
            return new CommonToken(Token.EOF);
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public String getSourceName() {
            return IntStream.UNKNOWN_SOURCE_NAME;
        }

        @Override
        public TokenFactory getTokenFactory() {
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
}

Example 45 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public TokenFactory getTokenFactory() {
            return CommonTokenFactory.DEFAULT;
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}