Examples with CharStream org.antlr.v4.runtime.CharStream used on opensource projects

Example 96 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TimeLexerSpeed method load_legacy_java_ascii.

public void load_legacy_java_ascii(String resourceName, int n) throws Exception {
    // keep refs around so we can average memory
    CharStream[] input = new CharStream[n];
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream[] streams = new InputStream[n];
    for (int i = 0; i < n; i++) {
        streams[i] = loader.getResourceAsStream(resourceName);
    }
    // track only time to suck data out of stream
    long start = System.nanoTime();
    for (int i = 0; i < n; i++) {
        try (InputStream is = streams[i]) {
            input[i] = new ANTLRInputStream(is);
        }
    }
    long stop = System.nanoTime();
    long tus = (stop - start) / 1000;
    int size = input[0].size();
    long streamSize = GraphLayout.parseInstance((Object) input[0]).totalSize();
    streamFootprints.add(basename(resourceName) + " (" + size + " char): " + GraphLayout.parseInstance((Object) input[0]).toFootprint());
    String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
    if (output)
        System.out.printf("%27s average time %5dus size %6db over %4d loads of %5d symbols from %s\n", currentMethodName, tus / n, streamSize, n, size, basename(resourceName));
}

Example 97 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TimeLexerSpeed method lex_new_java_utf8.

public void lex_new_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    try (InputStream is = loader.getResourceAsStream(Parser_java_file)) {
        long size = getResourceSize(loader, Parser_java_file);
        CharStream input = CharStreams.fromStream(is, StandardCharsets.UTF_8, size);
        JavaLexer lexer = new JavaLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
    }
}

Example 98 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TimeLexerSpeed method load_new_utf8.

public void load_new_utf8(String resourceName, int n) throws Exception {
    // keep refs around so we can average memory
    CharStream[] input = new CharStream[n];
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream[] streams = new InputStream[n];
    for (int i = 0; i < n; i++) {
        streams[i] = loader.getResourceAsStream(resourceName);
    }
    URLConnection uc = null;
    long streamLength = getResourceSize(loader, resourceName);
    // track only time to suck data out of stream
    long start = System.nanoTime();
    for (int i = 0; i < n; i++) {
        try (InputStream is = streams[i]) {
            input[i] = CharStreams.fromStream(is, StandardCharsets.UTF_8, streamLength);
        }
    }
    long stop = System.nanoTime();
    long tus = (stop - start) / 1000;
    int size = input[0].size();
    long streamSize = GraphLayout.parseInstance((Object) input[0]).totalSize();
    streamFootprints.add(basename(resourceName) + " (" + size + " char): " + GraphLayout.parseInstance((Object) input[0]).toFootprint());
    String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
    if (output)
        System.out.printf("%27s average time %5dus size %6db over %4d loads of %5d symbols from %s\n", currentMethodName, tus / n, streamSize, n, size, basename(resourceName));
}

Example 99 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TimeLexerSpeed method lex_new_grapheme_utf8.

public void lex_new_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    String resourceName = PerfDir + "/" + fileName;
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    try (InputStream is = loader.getResourceAsStream(resourceName)) {
        long size = getResourceSize(loader, resourceName);
        CharStream input = CharStreams.fromStream(is, StandardCharsets.UTF_8, size);
        graphemesLexer lexer = new graphemesLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
    }
}

Example 100 with CharStream

use of org.antlr.v4.runtime.CharStream in project antlr4 by antlr.

the class TestCharStreams method fromReader.

@Test
public void fromReader() throws Exception {
    Path p = folder.newFile().toPath();
    Files.write(p, "hello \uD83C\uDF0E".getBytes(StandardCharsets.UTF_8));
    try (Reader r = Files.newBufferedReader(p, StandardCharsets.UTF_8)) {
        CharStream s = CharStreams.fromReader(r);
        assertEquals(7, s.size());
        assertEquals(0, s.index());
        assertEquals("hello \uD83C\uDF0E", s.toString());
    }
}