Examples with TokenSource org.antlr.v4.runtime.TokenSource used on opensource projects

Example 16 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestPerformance method getParserFactory.

protected ParserFactory getParserFactory(String lexerName, String parserName, String listenerName, final String entryPoint) {
    try {
        ClassLoader loader = new URLClassLoader(new URL[] { getTempTestDir().toURI().toURL() }, ClassLoader.getSystemClassLoader());
        final Class<? extends Lexer> lexerClass = loader.loadClass(lexerName).asSubclass(Lexer.class);
        final Class<? extends Parser> parserClass = loader.loadClass(parserName).asSubclass(Parser.class);
        final Class<? extends ParseTreeListener> listenerClass = loader.loadClass(listenerName).asSubclass(ParseTreeListener.class);
        final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
        final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
        // construct initial instances of the lexer and parser to deserialize their ATNs
        TokenSource tokenSource = lexerCtor.newInstance(new ANTLRInputStream(""));
        parserCtor.newInstance(new CommonTokenStream(tokenSource));
        return new ParserFactory() {

            @Override
            public FileParseResult parseFile(CharStream input, int currentPass, int thread) {
                final MurmurHashChecksum checksum = new MurmurHashChecksum();
                final long startTime = System.nanoTime();
                assert thread >= 0 && thread < NUMBER_OF_THREADS;
                try {
                    ParseTreeListener listener = sharedListeners[thread];
                    if (listener == null) {
                        listener = listenerClass.newInstance();
                        sharedListeners[thread] = listener;
                    }
                    Lexer lexer = sharedLexers[thread];
                    if (REUSE_LEXER && lexer != null) {
                        lexer.setInputStream(input);
                    } else {
                        Lexer previousLexer = lexer;
                        lexer = lexerCtor.newInstance(input);
                        DFA[] decisionToDFA = (FILE_GRANULARITY || previousLexer == null ? lexer : previousLexer).getInterpreter().decisionToDFA;
                        if (!REUSE_LEXER_DFA || (!FILE_GRANULARITY && previousLexer == null)) {
                            decisionToDFA = new DFA[decisionToDFA.length];
                        }
                        if (COMPUTE_TRANSITION_STATS) {
                            lexer.setInterpreter(new StatisticsLexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
                        } else if (!REUSE_LEXER_DFA) {
                            lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
                        }
                        sharedLexers[thread] = lexer;
                    }
                    lexer.removeErrorListeners();
                    lexer.addErrorListener(DescriptiveErrorListener.INSTANCE);
                    if (lexer.getInterpreter().decisionToDFA[0] == null) {
                        ATN atn = lexer.getATN();
                        for (int i = 0; i < lexer.getInterpreter().decisionToDFA.length; i++) {
                            lexer.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
                        }
                    }
                    CommonTokenStream tokens = new CommonTokenStream(lexer);
                    tokens.fill();
                    tokenCount.addAndGet(currentPass, tokens.size());
                    if (COMPUTE_CHECKSUM) {
                        for (Token token : tokens.getTokens()) {
                            updateChecksum(checksum, token);
                        }
                    }
                    if (!RUN_PARSER) {
                        return new FileParseResult(input.getSourceName(), (int) checksum.getValue(), null, tokens.size(), startTime, lexer, null);
                    }
                    final long parseStartTime = System.nanoTime();
                    Parser parser = sharedParsers[thread];
                    if (REUSE_PARSER && parser != null) {
                        parser.setInputStream(tokens);
                    } else {
                        Parser previousParser = parser;
                        if (USE_PARSER_INTERPRETER) {
                            Parser referenceParser = parserCtor.newInstance(tokens);
                            parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                        } else {
                            parser = parserCtor.newInstance(tokens);
                        }
                        DFA[] decisionToDFA = (FILE_GRANULARITY || previousParser == null ? parser : previousParser).getInterpreter().decisionToDFA;
                        if (!REUSE_PARSER_DFA || (!FILE_GRANULARITY && previousParser == null)) {
                            decisionToDFA = new DFA[decisionToDFA.length];
                        }
                        if (COMPUTE_TRANSITION_STATS) {
                            parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                        } else if (!REUSE_PARSER_DFA) {
                            parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                        }
                        sharedParsers[thread] = parser;
                    }
                    parser.removeParseListeners();
                    parser.removeErrorListeners();
                    if (!TWO_STAGE_PARSING) {
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                    }
                    if (parser.getInterpreter().decisionToDFA[0] == null) {
                        ATN atn = parser.getATN();
                        for (int i = 0; i < parser.getInterpreter().decisionToDFA.length; i++) {
                            parser.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
                        }
                    }
                    parser.getInterpreter().setPredictionMode(TWO_STAGE_PARSING ? PredictionMode.SLL : PREDICTION_MODE);
                    parser.setBuildParseTree(BUILD_PARSE_TREES);
                    if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                        parser.addParseListener(listener);
                    }
                    if (BAIL_ON_ERROR || TWO_STAGE_PARSING) {
                        parser.setErrorHandler(new BailErrorStrategy());
                    }
                    Method parseMethod = parserClass.getMethod(entryPoint);
                    Object parseResult;
                    try {
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (USE_PARSER_INTERPRETER) {
                            ParserInterpreter parserInterpreter = (ParserInterpreter) parser;
                            parseResult = parserInterpreter.parse(Collections.lastIndexOfSubList(Arrays.asList(parser.getRuleNames()), Collections.singletonList(entryPoint)));
                        } else {
                            parseResult = parseMethod.invoke(parser);
                        }
                    } catch (InvocationTargetException ex) {
                        if (!TWO_STAGE_PARSING) {
                            throw ex;
                        }
                        String sourceName = tokens.getSourceName();
                        sourceName = sourceName != null && !sourceName.isEmpty() ? sourceName + ": " : "";
                        if (REPORT_SECOND_STAGE_RETRY) {
                            System.err.println(sourceName + "Forced to retry with full context.");
                        }
                        if (!(ex.getCause() instanceof ParseCancellationException)) {
                            throw ex;
                        }
                        tokens.seek(0);
                        if (REUSE_PARSER && parser != null) {
                            parser.setInputStream(tokens);
                        } else {
                            Parser previousParser = parser;
                            if (USE_PARSER_INTERPRETER) {
                                Parser referenceParser = parserCtor.newInstance(tokens);
                                parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                            } else {
                                parser = parserCtor.newInstance(tokens);
                            }
                            DFA[] decisionToDFA = previousParser.getInterpreter().decisionToDFA;
                            if (COMPUTE_TRANSITION_STATS) {
                                parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                            } else if (!REUSE_PARSER_DFA) {
                                parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                            }
                            sharedParsers[thread] = parser;
                        }
                        parser.removeParseListeners();
                        parser.removeErrorListeners();
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                        parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                        parser.setBuildParseTree(BUILD_PARSE_TREES);
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                            parser.addParseListener(listener);
                        }
                        if (BAIL_ON_ERROR) {
                            parser.setErrorHandler(new BailErrorStrategy());
                        }
                        parseResult = parseMethod.invoke(parser);
                    }
                    assertThat(parseResult, instanceOf(ParseTree.class));
                    if (COMPUTE_CHECKSUM && BUILD_PARSE_TREES) {
                        ParseTreeWalker.DEFAULT.walk(new ChecksumParseTreeListener(checksum), (ParseTree) parseResult);
                    }
                    if (BUILD_PARSE_TREES && BLANK_LISTENER) {
                        ParseTreeWalker.DEFAULT.walk(listener, (ParseTree) parseResult);
                    }
                    return new FileParseResult(input.getSourceName(), (int) checksum.getValue(), (ParseTree) parseResult, tokens.size(), TIME_PARSE_ONLY ? parseStartTime : startTime, lexer, parser);
                } catch (Exception e) {
                    if (!REPORT_SYNTAX_ERRORS && e instanceof ParseCancellationException) {
                        return new FileParseResult("unknown", (int) checksum.getValue(), null, 0, startTime, null, null);
                    }
                    e.printStackTrace(System.out);
                    throw new IllegalStateException(e);
                }
            }
        };
    } catch (Exception e) {
        e.printStackTrace(System.out);
        Assert.fail(e.getMessage());
        throw new IllegalStateException(e);
    }
}

Example 17 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 18 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}