Examples with CharStream org.antlr.v4.runtime.CharStream used on opensource projects

Example 31 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_legacy_grapheme_utf8.

public void lex_legacy_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(PerfDir + "/" + fileName);
    try {
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
        try {
            BufferedReader br = new BufferedReader(isr);
            try {
                @SuppressWarnings("deprecation") CharStream input = new org.antlr.v4.runtime.ANTLRInputStream(br);
                graphemesLexer lexer = new graphemesLexer(input);
                double avg = tokenize(lexer, n, clearLexerDFACache);
                String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
                if (output)
                    System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
            } finally {
                br.close();
            }
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
}

Example 32 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestATNLexerInterpreter method checkLexerMatches.

protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) {
    ATN atn = createATN(lg, true);
    CharStream input = CharStreams.fromString(inputString);
    ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE");
    DOTGenerator dot = new DOTGenerator(lg);
    System.out.println(dot.getDOT(startState, true));
    List<String> tokenTypes = getTokenTypes(lg, atn, input);
    String result = Utils.join(tokenTypes.iterator(), ", ");
    System.out.println(tokenTypes);
    assertEquals(expecting, result);
}

Example 33 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class BaseTest method getTokenTypes.

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}

Example 34 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestBufferedTokenStream method testFirstToken.

@Test
public void testFirstToken() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = CharStreams.fromString("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    String result = tokens.LT(1).getText();
    String expecting = "x";
    assertEquals(expecting, result);
}

Example 35 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by tunnelvisionlabs.

the class TestCharStreams method fromSMPUTF16LEPathSMPHasExpectedSize.

@Test
public void fromSMPUTF16LEPathSMPHasExpectedSize() throws Exception {
    File p = folder.newFile();
    Utils.writeFile(p, "hello \uD83C\uDF0E".getBytes(Charset.forName("UTF-16LE")));
    CharStream s = CharStreams.fromFile(p, Charset.forName("UTF-16LE"));
    assertEquals(7, s.size());
    assertEquals(0, s.index());
    assertEquals("hello \uD83C\uDF0E", s.toString());
    assertEquals(p.toString(), s.getSourceName());
}