Examples with CharStream org.antlr.v4.runtime.CharStream used on opensource projects

Example 61 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestUnbufferedCharStream method testNestedMarkReleasedTwice.

/**
 * The {@link IntStream} interface does not specify the behavior when a mark
 * is released twice, but {@link UnbufferedCharStream} handles this case by
 * throwing an {@link IllegalStateException}.
 */
@Test(expected = IllegalStateException.class)
public void testNestedMarkReleasedTwice() {
    CharStream input = createStream("");
    int m1 = input.mark();
    int m2 = input.mark();
    input.release(m2);
    input.release(m2);
}

Example 62 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestUnbufferedCharStream method testMarkReleasedTwice.

/**
 * The {@link IntStream} interface does not specify the behavior when a mark
 * is released twice, but {@link UnbufferedCharStream} handles this case by
 * throwing an {@link IllegalStateException}.
 */
@Test(expected = IllegalStateException.class)
public void testMarkReleasedTwice() {
    CharStream input = createStream("");
    int m1 = input.mark();
    input.release(m1);
    input.release(m1);
}

Example 63 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestUnbufferedTokenStream method testLookahead.

@Test
public void testLookahead() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 302;
    CharStream input = CharStreams.fromString("x = 302;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = new UnbufferedTokenStream(lexEngine);
    assertEquals("x", tokens.LT(1).getText());
    assertEquals(" ", tokens.LT(2).getText());
    assertEquals("=", tokens.LT(3).getText());
    assertEquals(" ", tokens.LT(4).getText());
    assertEquals("302", tokens.LT(5).getText());
    assertEquals(";", tokens.LT(6).getText());
}

Example 64 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestUnicodeGrammar method binaryGrammar.

@Test
public void binaryGrammar() throws Exception {
    String grammarText = "grammar Binary;\n" + "r : HEADER PACKET+ FOOTER;\n" + "HEADER : '\\u0002\\u0000\\u0001\\u0007';\n" + "PACKET : '\\u00D0' ('\\u00D1' | '\\u00D2' | '\\u00D3') +;\n" + "FOOTER : '\\u00FF';\n";
    byte[] toParse = new byte[] { (byte) 0x02, (byte) 0x00, (byte) 0x01, (byte) 0x07, (byte) 0xD0, (byte) 0xD2, (byte) 0xD2, (byte) 0xD3, (byte) 0xD3, (byte) 0xD3, (byte) 0xD0, (byte) 0xD3, (byte) 0xD3, (byte) 0xD1, (byte) 0xFF };
    CharStream charStream;
    ByteArrayInputStream is = new ByteArrayInputStream(toParse);
    try {
        // Note we use ISO_8859_1 to treat all byte values as Unicode "characters" from
        // U+0000 to U+00FF.
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("ISO-8859-1"));
        try {
            charStream = CharStreams.fromReader(isr);
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
    Grammar grammar = new Grammar(grammarText);
    LexerInterpreter lexEngine = grammar.createLexerInterpreter(charStream);
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens);
    ParseTree parseTree = parser.parse(grammar.rules.get("r").index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(grammar.getRuleNames());
    String result = Trees.toStringTree(parseTree, nodeTextProvider);
    assertEquals("(r:1 \u0002\u0000\u0001\u0007 \u00D0\u00D2\u00D2\u00D3\u00D3\u00D3 \u00D0\u00D3\u00D3\u00D1 \u00FF)", result);
}

Example 65 with CharStream

use of org.antlr.v4.runtime.CharStream in project presto by prestodb.

the class StatementSplitter method getLexer.

private static TokenSource getLexer(String sql, Set<String> terminators) {
    requireNonNull(sql, "sql is null");
    CharStream stream = new CaseInsensitiveStream(new ANTLRInputStream(sql));
    return new DelimiterLexer(stream, terminators);
}