Examples with ParserInterpreter org.antlr.v4.runtime.ParserInterpreter used on opensource projects

Example 1 with ParserInterpreter

use of org.antlr.v4.runtime.ParserInterpreter in project antlr4 by tunnelvisionlabs.

the class TestAmbigParseTrees method testInterpAtSpecificAlt.

void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g, String startRule, int startAlt, String input, String expectedParseTree) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(CharStreams.fromString(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
    Transition tr = ruleStartState.transition(0);
    ATNState t2 = tr.target;
    if (!(t2 instanceof BasicBlockStartState)) {
        throw new IllegalArgumentException("rule has no decision: " + startRule);
    }
    parser.addDecisionOverride(((DecisionState) t2).decision, 0, startAlt);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
}

Example 2 with ParserInterpreter

use of org.antlr.v4.runtime.ParserInterpreter in project antlr4 by tunnelvisionlabs.

the class GrammarParserInterpreter method getLookaheadParseTrees.

/**
 * Return a list of parse trees, one for each alternative in a decision
 *  given the same input.
 *
 *  Very similar to {@link #getAllPossibleParseTrees} except
 *  that it re-parses the input for every alternative in a decision,
 *  not just the ambiguous ones (there is no alts parameter here).
 *  This method also tries to reduce the size of the parse trees
 *  by stripping away children of the tree that are completely out of range
 *  of startIndex..stopIndex. Also, because errors are expected, we
 *  use a specialized error handler that more or less bails out
 *  but that also consumes the first erroneous token at least. This
 *  ensures that an error node will be in the parse tree for display.
 *
 *  NOTES:
 *    // we must parse the entire input now with decision overrides
 *	// we cannot parse a subset because it could be that a decision
 *	// above our decision of interest needs to read way past
 *	// lookaheadInfo.stopIndex. It seems like there is no escaping
 *	// the use of a full and complete token stream if we are
 *	// resetting to token index 0 and re-parsing from the start symbol.
 *	// It's not easy to restart parsing somewhere in the middle like a
 *	// continuation because our call stack does not match the
 *	// tree stack because of left recursive rule rewriting. grrrr!
 *
 * @since 4.5.1
 */
public static List<ParserRuleContext> getLookaheadParseTrees(Grammar g, ParserInterpreter originalParser, TokenStream tokens, int startRuleIndex, int decision, int startIndex, int stopIndex) {
    List<ParserRuleContext> trees = new ArrayList<ParserRuleContext>();
    // Create a new parser interpreter to parse the ambiguous subphrase
    ParserInterpreter parser = deriveTempParserInterpreter(g, originalParser, tokens);
    DecisionState decisionState = originalParser.getATN().decisionToState.get(decision);
    for (int alt = 1; alt <= decisionState.getTransitions().length; alt++) {
        // re-parse entire input for all ambiguous alternatives
        // (don't have to do first as it's been parsed, but do again for simplicity
        // using this temp parser.)
        GrammarParserInterpreter.BailButConsumeErrorStrategy errorHandler = new GrammarParserInterpreter.BailButConsumeErrorStrategy();
        parser.setErrorHandler(errorHandler);
        parser.reset();
        parser.addDecisionOverride(decision, startIndex, alt);
        ParserRuleContext tt = parser.parse(startRuleIndex);
        int stopTreeAt = stopIndex;
        if (errorHandler.firstErrorTokenIndex >= 0) {
            // cut off rest at first error
            stopTreeAt = errorHandler.firstErrorTokenIndex;
        }
        Interval overallRange = tt.getSourceInterval();
        if (stopTreeAt > overallRange.b) {
            // If we try to look beyond range of tree, stopTreeAt must be EOF
            // for which there is no EOF ref in grammar. That means tree
            // will not have node for stopTreeAt; limit to overallRange.b
            stopTreeAt = overallRange.b;
        }
        ParserRuleContext subtree = Trees.getRootOfSubtreeEnclosingRegion(tt, startIndex, stopTreeAt);
        // Use higher of overridden decision tree or tree enclosing all tokens
        if (Trees.isAncestorOf(parser.getOverrideDecisionRoot(), subtree)) {
            subtree = parser.getOverrideDecisionRoot();
        }
        Trees.stripChildrenOutOfRange(subtree, parser.getOverrideDecisionRoot(), startIndex, stopTreeAt);
        trees.add(subtree);
    }
    return trees;
}

Example 3 with ParserInterpreter

use of org.antlr.v4.runtime.ParserInterpreter in project antlr4 by tunnelvisionlabs.

the class ParseTreePatternMatcher method compile.

/**
 * For repeated use of a tree pattern, compile it to a
 * {@link ParseTreePattern} using this method.
 */
public ParseTreePattern compile(String pattern, int patternRuleIndex) {
    List<? extends Token> tokenList = tokenize(pattern);
    ListTokenSource tokenSrc = new ListTokenSource(tokenList);
    CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
    ParserInterpreter parserInterp = new ParserInterpreter(parser.getGrammarFileName(), parser.getVocabulary(), Arrays.asList(parser.getRuleNames()), parser.getATNWithBypassAlts(), tokens);
    ParseTree tree = null;
    try {
        parserInterp.setErrorHandler(new BailErrorStrategy());
        tree = parserInterp.parse(patternRuleIndex);
    // System.out.println("pattern tree = "+tree.toStringTree(parserInterp));
    } catch (ParseCancellationException e) {
        throw (RecognitionException) e.getCause();
    } catch (RecognitionException re) {
        throw re;
    } catch (Exception e) {
        throw new CannotInvokeStartRule(e);
    }
    // Make sure tree pattern compilation checks for a complete parse
    if (tokens.LA(1) != Token.EOF) {
        throw new StartRuleDoesNotConsumeFullPattern();
    }
    return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
}

Example 4 with ParserInterpreter

use of org.antlr.v4.runtime.ParserInterpreter in project antlr4 by tunnelvisionlabs.

the class TestPerformance method getParserFactory.

protected ParserFactory getParserFactory(String lexerName, String parserName, String listenerName, final String entryPoint) {
    try {
        ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader());
        final Class<? extends Lexer> lexerClass = loader.loadClass(lexerName).asSubclass(Lexer.class);
        final Class<? extends Parser> parserClass = loader.loadClass(parserName).asSubclass(Parser.class);
        final Class<? extends ParseTreeListener> listenerClass = (Class<? extends ParseTreeListener>) loader.loadClass(listenerName).asSubclass(ParseTreeListener.class);
        final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
        final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
        // construct initial instances of the lexer and parser to deserialize their ATNs
        TokenSource tokenSource = lexerCtor.newInstance(CharStreams.fromString(""));
        parserCtor.newInstance(new CommonTokenStream(tokenSource));
        if (!REUSE_LEXER_DFA) {
            Field lexerSerializedATNField = lexerClass.getField("_serializedATN");
            String lexerSerializedATN = (String) lexerSerializedATNField.get(null);
            for (int i = 0; i < NUMBER_OF_THREADS; i++) {
                sharedLexerATNs[i] = new ATNDeserializer().deserialize(lexerSerializedATN.toCharArray());
            }
        }
        if (RUN_PARSER && !REUSE_PARSER_DFA) {
            Field parserSerializedATNField = parserClass.getField("_serializedATN");
            String parserSerializedATN = (String) parserSerializedATNField.get(null);
            for (int i = 0; i < NUMBER_OF_THREADS; i++) {
                sharedParserATNs[i] = new ATNDeserializer().deserialize(parserSerializedATN.toCharArray());
            }
        }
        return new ParserFactory() {

            @SuppressWarnings("unused")
            @Override
            public FileParseResult parseFile(CharStream input, int currentPass, int thread) {
                final MurmurHashChecksum checksum = new MurmurHashChecksum();
                final long startTime = System.nanoTime();
                assert thread >= 0 && thread < NUMBER_OF_THREADS;
                try {
                    ParseTreeListener listener = sharedListeners[thread];
                    if (listener == null) {
                        listener = listenerClass.newInstance();
                        sharedListeners[thread] = listener;
                    }
                    Lexer lexer = sharedLexers[thread];
                    if (REUSE_LEXER && lexer != null) {
                        lexer.setInputStream(input);
                    } else {
                        Lexer previousLexer = lexer;
                        lexer = lexerCtor.newInstance(input);
                        sharedLexers[thread] = lexer;
                        ATN atn = (FILE_GRANULARITY || previousLexer == null ? lexer : previousLexer).getATN();
                        if (!REUSE_LEXER_DFA || (!FILE_GRANULARITY && previousLexer == null)) {
                            atn = sharedLexerATNs[thread];
                        }
                        if (!ENABLE_LEXER_DFA) {
                            lexer.setInterpreter(new NonCachingLexerATNSimulator(lexer, atn));
                        } else if (!REUSE_LEXER_DFA || COMPUTE_TRANSITION_STATS) {
                            lexer.setInterpreter(new StatisticsLexerATNSimulator(lexer, atn));
                        }
                    }
                    lexer.removeErrorListeners();
                    lexer.addErrorListener(DescriptiveLexerErrorListener.INSTANCE);
                    lexer.getInterpreter().optimize_tail_calls = OPTIMIZE_TAIL_CALLS;
                    if (ENABLE_LEXER_DFA && !REUSE_LEXER_DFA) {
                        lexer.getInterpreter().atn.clearDFA();
                    }
                    CommonTokenStream tokens = new CommonTokenStream(lexer);
                    tokens.fill();
                    tokenCount.addAndGet(currentPass, tokens.size());
                    if (COMPUTE_CHECKSUM) {
                        for (Token token : tokens.getTokens()) {
                            updateChecksum(checksum, token);
                        }
                    }
                    if (!RUN_PARSER) {
                        return new FileParseResult(input.getSourceName(), checksum.getValue(), null, tokens.size(), startTime, lexer, null);
                    }
                    final long parseStartTime = System.nanoTime();
                    Parser parser = sharedParsers[thread];
                    if (REUSE_PARSER && parser != null) {
                        parser.setInputStream(tokens);
                    } else {
                        Parser previousParser = parser;
                        if (USE_PARSER_INTERPRETER) {
                            Parser referenceParser = parserCtor.newInstance(tokens);
                            parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                        } else {
                            parser = parserCtor.newInstance(tokens);
                        }
                        ATN atn = (FILE_GRANULARITY || previousParser == null ? parser : previousParser).getATN();
                        if (!REUSE_PARSER_DFA || (!FILE_GRANULARITY && previousParser == null)) {
                            atn = sharedLexerATNs[thread];
                        }
                        if (!ENABLE_PARSER_DFA) {
                            parser.setInterpreter(new NonCachingParserATNSimulator(parser, atn));
                        } else if (!REUSE_PARSER_DFA || COMPUTE_TRANSITION_STATS) {
                            parser.setInterpreter(new StatisticsParserATNSimulator(parser, atn));
                        }
                        sharedParsers[thread] = parser;
                    }
                    parser.removeParseListeners();
                    parser.removeErrorListeners();
                    if (!TWO_STAGE_PARSING) {
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                    }
                    if (ENABLE_PARSER_DFA && !REUSE_PARSER_DFA) {
                        parser.getInterpreter().atn.clearDFA();
                    }
                    parser.getInterpreter().setPredictionMode(TWO_STAGE_PARSING ? PredictionMode.SLL : PREDICTION_MODE);
                    parser.getInterpreter().force_global_context = FORCE_GLOBAL_CONTEXT && !TWO_STAGE_PARSING;
                    parser.getInterpreter().always_try_local_context = TRY_LOCAL_CONTEXT_FIRST || TWO_STAGE_PARSING;
                    parser.getInterpreter().enable_global_context_dfa = ENABLE_PARSER_FULL_CONTEXT_DFA;
                    parser.getInterpreter().optimize_ll1 = OPTIMIZE_LL1;
                    parser.getInterpreter().optimize_unique_closure = OPTIMIZE_UNIQUE_CLOSURE;
                    parser.getInterpreter().optimize_tail_calls = OPTIMIZE_TAIL_CALLS;
                    parser.getInterpreter().tail_call_preserves_sll = TAIL_CALL_PRESERVES_SLL;
                    parser.getInterpreter().treat_sllk1_conflict_as_ambiguity = TREAT_SLLK1_CONFLICT_AS_AMBIGUITY;
                    parser.setBuildParseTree(BUILD_PARSE_TREES);
                    if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                        parser.addParseListener(listener);
                    }
                    if (BAIL_ON_ERROR || TWO_STAGE_PARSING) {
                        parser.setErrorHandler(new BailErrorStrategy());
                    }
                    Method parseMethod = parserClass.getMethod(entryPoint);
                    Object parseResult;
                    try {
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (USE_PARSER_INTERPRETER) {
                            ParserInterpreter parserInterpreter = (ParserInterpreter) parser;
                            parseResult = parserInterpreter.parse(Collections.lastIndexOfSubList(Arrays.asList(parser.getRuleNames()), Collections.singletonList(entryPoint)));
                        } else {
                            parseResult = parseMethod.invoke(parser);
                        }
                    } catch (InvocationTargetException ex) {
                        if (!TWO_STAGE_PARSING) {
                            throw ex;
                        }
                        String sourceName = tokens.getSourceName();
                        sourceName = sourceName != null && !sourceName.isEmpty() ? sourceName + ": " : "";
                        if (REPORT_SECOND_STAGE_RETRY) {
                            System.err.println(sourceName + "Forced to retry with full context.");
                        }
                        if (!(ex.getCause() instanceof ParseCancellationException)) {
                            throw ex;
                        }
                        tokens.seek(0);
                        if (REUSE_PARSER && sharedParsers[thread] != null) {
                            parser.setInputStream(tokens);
                        } else {
                            if (USE_PARSER_INTERPRETER) {
                                Parser referenceParser = parserCtor.newInstance(tokens);
                                parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                            } else {
                                parser = parserCtor.newInstance(tokens);
                            }
                            sharedParsers[thread] = parser;
                        }
                        parser.removeParseListeners();
                        parser.removeErrorListeners();
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                        if (!ENABLE_PARSER_DFA) {
                            parser.setInterpreter(new NonCachingParserATNSimulator(parser, parser.getATN()));
                        } else if (!REUSE_PARSER_DFA) {
                            parser.setInterpreter(new StatisticsParserATNSimulator(parser, sharedParserATNs[thread]));
                        } else if (COMPUTE_TRANSITION_STATS) {
                            parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN()));
                        }
                        parser.getInterpreter().setPredictionMode(PREDICTION_MODE);
                        parser.getInterpreter().force_global_context = FORCE_GLOBAL_CONTEXT;
                        parser.getInterpreter().always_try_local_context = TRY_LOCAL_CONTEXT_FIRST;
                        parser.getInterpreter().enable_global_context_dfa = ENABLE_PARSER_FULL_CONTEXT_DFA;
                        parser.getInterpreter().optimize_ll1 = OPTIMIZE_LL1;
                        parser.getInterpreter().optimize_unique_closure = OPTIMIZE_UNIQUE_CLOSURE;
                        parser.getInterpreter().optimize_tail_calls = OPTIMIZE_TAIL_CALLS;
                        parser.getInterpreter().tail_call_preserves_sll = TAIL_CALL_PRESERVES_SLL;
                        parser.getInterpreter().treat_sllk1_conflict_as_ambiguity = TREAT_SLLK1_CONFLICT_AS_AMBIGUITY;
                        parser.setBuildParseTree(BUILD_PARSE_TREES);
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                            parser.addParseListener(listener);
                        }
                        if (BAIL_ON_ERROR) {
                            parser.setErrorHandler(new BailErrorStrategy());
                        }
                        parseResult = parseMethod.invoke(parser);
                    }
                    assertThat(parseResult, instanceOf(ParseTree.class));
                    if (COMPUTE_CHECKSUM && BUILD_PARSE_TREES) {
                        ParseTreeWalker.DEFAULT.walk(new ChecksumParseTreeListener(checksum), (ParseTree) parseResult);
                    }
                    if (BUILD_PARSE_TREES && BLANK_LISTENER) {
                        ParseTreeWalker.DEFAULT.walk(listener, (ParserRuleContext) parseResult);
                    }
                    return new FileParseResult(input.getSourceName(), checksum.getValue(), (ParseTree) parseResult, tokens.size(), TIME_PARSE_ONLY ? parseStartTime : startTime, lexer, parser);
                } catch (Exception e) {
                    if (!REPORT_SYNTAX_ERRORS && e instanceof ParseCancellationException) {
                        return new FileParseResult("unknown", checksum.getValue(), null, 0, startTime, null, null);
                    }
                    e.printStackTrace(System.out);
                    throw new IllegalStateException(e);
                }
            }
        };
    } catch (Exception e) {
        e.printStackTrace(System.out);
        Assert.fail(e.getMessage());
        throw new IllegalStateException(e);
    }
}

Example 5 with ParserInterpreter

use of org.antlr.v4.runtime.ParserInterpreter in project antlr4 by antlr.

the class TestParserProfiler method interpAndGetDecisionInfo.

public DecisionInfo[] interpAndGetDecisionInfo(LexerGrammar lg, Grammar g, String startRule, String... input) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(null);
    ParserInterpreter parser = g.createParserInterpreter(null);
    parser.setProfile(true);
    for (String s : input) {
        lexEngine.reset();
        parser.reset();
        lexEngine.setInputStream(new ANTLRInputStream(s));
        CommonTokenStream tokens = new CommonTokenStream(lexEngine);
        parser.setInputStream(tokens);
        Rule r = g.rules.get(startRule);
        if (r == null) {
            return parser.getParseInfo().getDecisionInfo();
        }
        ParserRuleContext t = parser.parse(r.index);
    // try {
    // Utils.waitForClose(t.inspect(parser).get());
    // }
    // catch (Exception e) {
    // e.printStackTrace();
    // }
    // 
    // System.out.println(t.toStringTree(parser));
    }
    return parser.getParseInfo().getDecisionInfo();
}