Examples with Lexer org.antlr.v4.runtime.Lexer used on opensource projects

Example 71 with Lexer

use of org.antlr.v4.runtime.Lexer in project drill by apache.

the class LogicalExpressionParser method parse.

/**
 * Initializes logical expression lexer and parser, add error listener that converts all
 * syntax error into {@link org.apache.drill.common.exceptions.ExpressionParsingException}.
 * Parses given expression into logical expression instance.
 *
 * @param expr expression to be parsed
 * @return logical expression instance
 */
public static LogicalExpression parse(String expr) {
    ExprLexer lexer = new ExprLexer(CharStreams.fromString(expr));
    // need to remove since default listener will output warning
    lexer.removeErrorListeners();
    lexer.addErrorListener(ErrorListener.INSTANCE);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ExprParser parser = new ExprParser(tokens);
    // need to remove since default listener will output warning
    parser.removeErrorListeners();
    parser.addErrorListener(ErrorListener.INSTANCE);
    ExprParser.ParseContext parseContext = parser.parse();
    logger.trace("Tokens: [{}]. Parsing details: [{}].", tokens.getText(), parseContext.toInfoString(parser));
    return parseContext.e;
}

Example 72 with Lexer

use of org.antlr.v4.runtime.Lexer in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_new_grapheme_utf8.

public void lex_new_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    String resourceName = PerfDir + "/" + fileName;
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream is = loader.getResourceAsStream(resourceName);
    try {
        long size = getResourceSize(loader, resourceName);
        CharStream input = CharStreams.fromStream(is, Charset.forName("UTF-8"), resourceName, size);
        graphemesLexer lexer = new graphemesLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
    } finally {
        is.close();
    }
}

Example 73 with Lexer

use of org.antlr.v4.runtime.Lexer in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method tokenize.

public double tokenize(Lexer lexer, int n, boolean clearLexerDFACache) {
    // always wipe the DFA before we begin tests so previous tests
    // don't affect this run!
    lexer.getInterpreter().clearDFA();
    long[] times = new long[n];
    for (int i = 0; i < n; i++) {
        lexer.reset();
        if (clearLexerDFACache) {
            lexer.getInterpreter().clearDFA();
        }
        long start = System.nanoTime();
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        // lex whole file.
        tokens.fill();
        // int size = lexer.getInputStream().size();
        long stop = System.nanoTime();
        times[i] = (stop - start) / 1000;
    // if ( output ) System.out.printf("Tokenized %d char in %dus\n", size, times[i]);
    }
    Arrays.sort(times);
    // drop highest 20% of times
    times = Arrays.copyOfRange(times, 0, times.length - (int) (n * .2));
    return avg(times);
}

Example 74 with Lexer

use of org.antlr.v4.runtime.Lexer in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_legacy_java_utf8.

public void lex_legacy_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(Parser_java_file);
    try {
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
        try {
            BufferedReader br = new BufferedReader(isr);
            try {
                @SuppressWarnings("deprecation") CharStream input = new org.antlr.v4.runtime.ANTLRInputStream(br);
                JavaLexer lexer = new JavaLexer(input);
                double avg = tokenize(lexer, n, clearLexerDFACache);
                String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
                if (output)
                    System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
            } finally {
                br.close();
            }
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
}

Example 75 with Lexer

use of org.antlr.v4.runtime.Lexer in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_legacy_grapheme_utf8.

public void lex_legacy_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(PerfDir + "/" + fileName);
    try {
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
        try {
            BufferedReader br = new BufferedReader(isr);
            try {
                @SuppressWarnings("deprecation") CharStream input = new org.antlr.v4.runtime.ANTLRInputStream(br);
                graphemesLexer lexer = new graphemesLexer(input);
                double avg = tokenize(lexer, n, clearLexerDFACache);
                String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
                if (output)
                    System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
            } finally {
                br.close();
            }
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
}