Examples with DFA org.antlr.v4.runtime.dfa.DFA used on opensource projects

Example 46 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by antlr.

the class LexerATNFactory method stringLiteral.

/**
 * For a lexer, a string is a sequence of char to match.  That is,
 *  "fog" is treated as 'f' 'o' 'g' not as a single transition in
 *  the DFA.  Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states
 *  for n characters.
 */
@Override
public Handle stringLiteral(TerminalAST stringLiteralAST) {
    String chars = stringLiteralAST.getText();
    ATNState left = newState(stringLiteralAST);
    ATNState right;
    String s = CharSupport.getStringFromGrammarStringLiteral(chars);
    if (s == null) {
        // the lexer will already have given an error
        return new Handle(left, left);
    }
    int n = s.length();
    ATNState prev = left;
    right = null;
    for (int i = 0; i < n; ) {
        right = newState(stringLiteralAST);
        int codePoint = s.codePointAt(i);
        prev.addTransition(CodePointTransitions.createWithCodePoint(right, codePoint));
        prev = right;
        i += Character.charCount(codePoint);
    }
    stringLiteralAST.atnState = left;
    return new Handle(left, right);
}

Example 47 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by antlr.

the class TimeLexerSpeed method lex_legacy_java_utf8.

public void lex_legacy_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    try (InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(Parser_java_file);
        InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
        BufferedReader br = new BufferedReader(isr)) {
        CharStream input = new ANTLRInputStream(br);
        JavaLexer lexer = new JavaLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
    }
}

Example 48 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by antlr.

the class TimeLexerSpeed method lex_new_java_utf8.

public void lex_new_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    try (InputStream is = loader.getResourceAsStream(Parser_java_file)) {
        long size = getResourceSize(loader, Parser_java_file);
        CharStream input = CharStreams.fromStream(is, StandardCharsets.UTF_8, size);
        JavaLexer lexer = new JavaLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
    }
}

Example 49 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by antlr.

the class TimeLexerSpeed method lex_new_grapheme_utf8.

public void lex_new_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    String resourceName = PerfDir + "/" + fileName;
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    try (InputStream is = loader.getResourceAsStream(resourceName)) {
        long size = getResourceSize(loader, resourceName);
        CharStream input = CharStreams.fromStream(is, StandardCharsets.UTF_8, size);
        graphemesLexer lexer = new graphemesLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
    }
}

Example 50 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by antlr.

the class RuntimeTestUtils method getTokenTypes.

public static List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}