Examples with DFA org.antlr.v4.runtime.dfa.DFA used on opensource projects

Example 11 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project sts4 by spring-projects.

the class AntlrParser method parse.

@Override
public ParseResults parse(String text) {
    ArrayList<Problem> syntaxErrors = new ArrayList<>();
    ArrayList<Problem> problems = new ArrayList<>();
    ArrayList<PropertiesAst.Node> astNodes = new ArrayList<>();
    JavaPropertiesLexer lexer = new JavaPropertiesLexer(new ANTLRInputStream(text.toCharArray(), text.length()));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    JavaPropertiesParser parser = new JavaPropertiesParser(tokens);
    // To avoid printing parse errors in the console
    parser.removeErrorListener(ConsoleErrorListener.INSTANCE);
    // Add listener to collect various parser errors
    parser.addErrorListener(new ANTLRErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            syntaxErrors.add(createProblem(msg, ProblemCodes.PROPERTIES_SYNTAX_ERROR, (Token) offendingSymbol));
        }

        @Override
        public void reportAmbiguity(org.antlr.v4.runtime.Parser recognizer, DFA dfa, int startIndex, int stopIndex, boolean exact, BitSet ambigAlts, ATNConfigSet configs) {
            problems.add(createProblem("Ambiguity detected!", ProblemCodes.PROPERTIES_AMBIGUITY_ERROR, recognizer.getCurrentToken()));
        }

        @Override
        public void reportAttemptingFullContext(org.antlr.v4.runtime.Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) {
            problems.add(createProblem("Full-Context attempt detected!", ProblemCodes.PROPERTIES_FULL_CONTEXT_ERROR, recognizer.getCurrentToken()));
        }

        @Override
        public void reportContextSensitivity(org.antlr.v4.runtime.Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, ATNConfigSet configs) {
            problems.add(createProblem("Context sensitivity detected!", ProblemCodes.PROPERTIES_CONTEXT_SENSITIVITY_ERROR, recognizer.getCurrentToken()));
        }
    });
    // Add listener to the parse tree to collect AST nodes
    parser.addParseListener(new JavaPropertiesBaseListener() {

        private Key key = null;

        private Value value = null;

        @Override
        public void exitPropertyLine(PropertyLineContext ctx) {
            KeyValuePair pair = new KeyValuePair(ctx, key, value);
            key.parent = value.parent = pair;
            astNodes.add(pair);
            key = null;
            value = null;
        }

        @Override
        public void exitCommentLine(CommentLineContext ctx) {
            astNodes.add(new Comment(ctx));
        }

        @Override
        public void exitKey(KeyContext ctx) {
            key = new Key(ctx);
        }

        @Override
        public void exitSeparatorAndValue(SeparatorAndValueContext ctx) {
            value = new Value(ctx);
        }

        @Override
        public void exitEmptyLine(EmptyLineContext ctx) {
            astNodes.add(new EmptyLine(ctx));
        }
    });
    parser.parse();
    // Collect and return parse results
    return new ParseResults(new PropertiesAst(ImmutableList.copyOf(astNodes)), ImmutableList.copyOf(syntaxErrors), ImmutableList.copyOf(problems));
}

Example 12 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_new_grapheme_utf8.

public void lex_new_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    String resourceName = PerfDir + "/" + fileName;
    ClassLoader loader = TimeLexerSpeed.class.getClassLoader();
    InputStream is = loader.getResourceAsStream(resourceName);
    try {
        long size = getResourceSize(loader, resourceName);
        CharStream input = CharStreams.fromStream(is, Charset.forName("UTF-8"), resourceName, size);
        graphemesLexer lexer = new graphemesLexer(input);
        double avg = tokenize(lexer, n, clearLexerDFACache);
        String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
        if (output)
            System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
    } finally {
        is.close();
    }
}

Example 13 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method tokenize.

public double tokenize(Lexer lexer, int n, boolean clearLexerDFACache) {
    // always wipe the DFA before we begin tests so previous tests
    // don't affect this run!
    lexer.getInterpreter().clearDFA();
    long[] times = new long[n];
    for (int i = 0; i < n; i++) {
        lexer.reset();
        if (clearLexerDFACache) {
            lexer.getInterpreter().clearDFA();
        }
        long start = System.nanoTime();
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        // lex whole file.
        tokens.fill();
        // int size = lexer.getInputStream().size();
        long stop = System.nanoTime();
        times[i] = (stop - start) / 1000;
    // if ( output ) System.out.printf("Tokenized %d char in %dus\n", size, times[i]);
    }
    Arrays.sort(times);
    // drop highest 20% of times
    times = Arrays.copyOfRange(times, 0, times.length - (int) (n * .2));
    return avg(times);
}

Example 14 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_legacy_java_utf8.

public void lex_legacy_java_utf8(int n, boolean clearLexerDFACache) throws Exception {
    InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(Parser_java_file);
    try {
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
        try {
            BufferedReader br = new BufferedReader(isr);
            try {
                @SuppressWarnings("deprecation") CharStream input = new org.antlr.v4.runtime.ANTLRInputStream(br);
                JavaLexer lexer = new JavaLexer(input);
                double avg = tokenize(lexer, n, clearLexerDFACache);
                String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
                if (output)
                    System.out.printf("%27s average time %5dus over %4d runs of %5d symbols%s\n", currentMethodName, (int) avg, n, input.size(), clearLexerDFACache ? " DFA cleared" : "");
            } finally {
                br.close();
            }
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
}

Example 15 with DFA

use of org.antlr.v4.runtime.dfa.DFA in project antlr4 by tunnelvisionlabs.

the class TimeLexerSpeed method lex_legacy_grapheme_utf8.

public void lex_legacy_grapheme_utf8(String fileName, int n, boolean clearLexerDFACache) throws Exception {
    InputStream is = TimeLexerSpeed.class.getClassLoader().getResourceAsStream(PerfDir + "/" + fileName);
    try {
        InputStreamReader isr = new InputStreamReader(is, Charset.forName("UTF-8"));
        try {
            BufferedReader br = new BufferedReader(isr);
            try {
                @SuppressWarnings("deprecation") CharStream input = new org.antlr.v4.runtime.ANTLRInputStream(br);
                graphemesLexer lexer = new graphemesLexer(input);
                double avg = tokenize(lexer, n, clearLexerDFACache);
                String currentMethodName = new Exception().getStackTrace()[0].getMethodName();
                if (output)
                    System.out.printf("%27s average time %5dus over %4d runs of %5d symbols from %s%s\n", currentMethodName, (int) avg, n, input.size(), fileName, clearLexerDFACache ? " DFA cleared" : "");
            } finally {
                br.close();
            }
        } finally {
            isr.close();
        }
    } finally {
        is.close();
    }
}