Examples with TokenSource org.antlr.v4.runtime.TokenSource used on opensource projects

Example 6 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 7 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}

Example 8 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project antlr4 by antlr.

the class TestPerformance method getParserFactory.

protected ParserFactory getParserFactory(String lexerName, String parserName, String listenerName, final String entryPoint) {
    try {
        ClassLoader loader = new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() }, ClassLoader.getSystemClassLoader());
        final Class<? extends Lexer> lexerClass = loader.loadClass(lexerName).asSubclass(Lexer.class);
        final Class<? extends Parser> parserClass = loader.loadClass(parserName).asSubclass(Parser.class);
        final Class<? extends ParseTreeListener> listenerClass = loader.loadClass(listenerName).asSubclass(ParseTreeListener.class);
        final Constructor<? extends Lexer> lexerCtor = lexerClass.getConstructor(CharStream.class);
        final Constructor<? extends Parser> parserCtor = parserClass.getConstructor(TokenStream.class);
        // construct initial instances of the lexer and parser to deserialize their ATNs
        TokenSource tokenSource = lexerCtor.newInstance(new ANTLRInputStream(""));
        parserCtor.newInstance(new CommonTokenStream(tokenSource));
        return new ParserFactory() {

            @Override
            public FileParseResult parseFile(CharStream input, int currentPass, int thread) {
                final MurmurHashChecksum checksum = new MurmurHashChecksum();
                final long startTime = System.nanoTime();
                assert thread >= 0 && thread < NUMBER_OF_THREADS;
                try {
                    ParseTreeListener listener = sharedListeners[thread];
                    if (listener == null) {
                        listener = listenerClass.newInstance();
                        sharedListeners[thread] = listener;
                    }
                    Lexer lexer = sharedLexers[thread];
                    if (REUSE_LEXER && lexer != null) {
                        lexer.setInputStream(input);
                    } else {
                        Lexer previousLexer = lexer;
                        lexer = lexerCtor.newInstance(input);
                        DFA[] decisionToDFA = (FILE_GRANULARITY || previousLexer == null ? lexer : previousLexer).getInterpreter().decisionToDFA;
                        if (!REUSE_LEXER_DFA || (!FILE_GRANULARITY && previousLexer == null)) {
                            decisionToDFA = new DFA[decisionToDFA.length];
                        }
                        if (COMPUTE_TRANSITION_STATS) {
                            lexer.setInterpreter(new StatisticsLexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
                        } else if (!REUSE_LEXER_DFA) {
                            lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), decisionToDFA, lexer.getInterpreter().getSharedContextCache()));
                        }
                        sharedLexers[thread] = lexer;
                    }
                    lexer.removeErrorListeners();
                    lexer.addErrorListener(DescriptiveErrorListener.INSTANCE);
                    if (lexer.getInterpreter().decisionToDFA[0] == null) {
                        ATN atn = lexer.getATN();
                        for (int i = 0; i < lexer.getInterpreter().decisionToDFA.length; i++) {
                            lexer.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
                        }
                    }
                    CommonTokenStream tokens = new CommonTokenStream(lexer);
                    tokens.fill();
                    tokenCount.addAndGet(currentPass, tokens.size());
                    if (COMPUTE_CHECKSUM) {
                        for (Token token : tokens.getTokens()) {
                            updateChecksum(checksum, token);
                        }
                    }
                    if (!RUN_PARSER) {
                        return new FileParseResult(input.getSourceName(), (int) checksum.getValue(), null, tokens.size(), startTime, lexer, null);
                    }
                    final long parseStartTime = System.nanoTime();
                    Parser parser = sharedParsers[thread];
                    if (REUSE_PARSER && parser != null) {
                        parser.setInputStream(tokens);
                    } else {
                        Parser previousParser = parser;
                        if (USE_PARSER_INTERPRETER) {
                            Parser referenceParser = parserCtor.newInstance(tokens);
                            parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                        } else {
                            parser = parserCtor.newInstance(tokens);
                        }
                        DFA[] decisionToDFA = (FILE_GRANULARITY || previousParser == null ? parser : previousParser).getInterpreter().decisionToDFA;
                        if (!REUSE_PARSER_DFA || (!FILE_GRANULARITY && previousParser == null)) {
                            decisionToDFA = new DFA[decisionToDFA.length];
                        }
                        if (COMPUTE_TRANSITION_STATS) {
                            parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                        } else if (!REUSE_PARSER_DFA) {
                            parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                        }
                        sharedParsers[thread] = parser;
                    }
                    parser.removeParseListeners();
                    parser.removeErrorListeners();
                    if (!TWO_STAGE_PARSING) {
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                    }
                    if (parser.getInterpreter().decisionToDFA[0] == null) {
                        ATN atn = parser.getATN();
                        for (int i = 0; i < parser.getInterpreter().decisionToDFA.length; i++) {
                            parser.getInterpreter().decisionToDFA[i] = new DFA(atn.getDecisionState(i), i);
                        }
                    }
                    parser.getInterpreter().setPredictionMode(TWO_STAGE_PARSING ? PredictionMode.SLL : PREDICTION_MODE);
                    parser.setBuildParseTree(BUILD_PARSE_TREES);
                    if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                        parser.addParseListener(listener);
                    }
                    if (BAIL_ON_ERROR || TWO_STAGE_PARSING) {
                        parser.setErrorHandler(new BailErrorStrategy());
                    }
                    Method parseMethod = parserClass.getMethod(entryPoint);
                    Object parseResult;
                    try {
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (USE_PARSER_INTERPRETER) {
                            ParserInterpreter parserInterpreter = (ParserInterpreter) parser;
                            parseResult = parserInterpreter.parse(Collections.lastIndexOfSubList(Arrays.asList(parser.getRuleNames()), Collections.singletonList(entryPoint)));
                        } else {
                            parseResult = parseMethod.invoke(parser);
                        }
                    } catch (InvocationTargetException ex) {
                        if (!TWO_STAGE_PARSING) {
                            throw ex;
                        }
                        String sourceName = tokens.getSourceName();
                        sourceName = sourceName != null && !sourceName.isEmpty() ? sourceName + ": " : "";
                        if (REPORT_SECOND_STAGE_RETRY) {
                            System.err.println(sourceName + "Forced to retry with full context.");
                        }
                        if (!(ex.getCause() instanceof ParseCancellationException)) {
                            throw ex;
                        }
                        tokens.seek(0);
                        if (REUSE_PARSER && parser != null) {
                            parser.setInputStream(tokens);
                        } else {
                            Parser previousParser = parser;
                            if (USE_PARSER_INTERPRETER) {
                                Parser referenceParser = parserCtor.newInstance(tokens);
                                parser = new ParserInterpreter(referenceParser.getGrammarFileName(), referenceParser.getVocabulary(), Arrays.asList(referenceParser.getRuleNames()), referenceParser.getATN(), tokens);
                            } else {
                                parser = parserCtor.newInstance(tokens);
                            }
                            DFA[] decisionToDFA = previousParser.getInterpreter().decisionToDFA;
                            if (COMPUTE_TRANSITION_STATS) {
                                parser.setInterpreter(new StatisticsParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                            } else if (!REUSE_PARSER_DFA) {
                                parser.setInterpreter(new ParserATNSimulator(parser, parser.getATN(), decisionToDFA, parser.getInterpreter().getSharedContextCache()));
                            }
                            sharedParsers[thread] = parser;
                        }
                        parser.removeParseListeners();
                        parser.removeErrorListeners();
                        parser.addErrorListener(DescriptiveErrorListener.INSTANCE);
                        parser.addErrorListener(new SummarizingDiagnosticErrorListener());
                        parser.getInterpreter().setPredictionMode(PredictionMode.LL);
                        parser.setBuildParseTree(BUILD_PARSE_TREES);
                        if (COMPUTE_CHECKSUM && !BUILD_PARSE_TREES) {
                            parser.addParseListener(new ChecksumParseTreeListener(checksum));
                        }
                        if (!BUILD_PARSE_TREES && BLANK_LISTENER) {
                            parser.addParseListener(listener);
                        }
                        if (BAIL_ON_ERROR) {
                            parser.setErrorHandler(new BailErrorStrategy());
                        }
                        parseResult = parseMethod.invoke(parser);
                    }
                    assertThat(parseResult, instanceOf(ParseTree.class));
                    if (COMPUTE_CHECKSUM && BUILD_PARSE_TREES) {
                        ParseTreeWalker.DEFAULT.walk(new ChecksumParseTreeListener(checksum), (ParseTree) parseResult);
                    }
                    if (BUILD_PARSE_TREES && BLANK_LISTENER) {
                        ParseTreeWalker.DEFAULT.walk(listener, (ParseTree) parseResult);
                    }
                    return new FileParseResult(input.getSourceName(), (int) checksum.getValue(), (ParseTree) parseResult, tokens.size(), TIME_PARSE_ONLY ? parseStartTime : startTime, lexer, parser);
                } catch (Exception e) {
                    if (!REPORT_SYNTAX_ERRORS && e instanceof ParseCancellationException) {
                        return new FileParseResult("unknown", (int) checksum.getValue(), null, 0, startTime, null, null);
                    }
                    e.printStackTrace(System.out);
                    throw new IllegalStateException(e);
                }
            }
        };
    } catch (Exception e) {
        e.printStackTrace(System.out);
        Assert.fail(e.getMessage());
        throw new IllegalStateException(e);
    }
}

Example 9 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project presto by prestodb.

the class StatementSplitter method squeezeStatement.

public static String squeezeStatement(String sql) {
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        } else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}

Example 10 with TokenSource

use of org.antlr.v4.runtime.TokenSource in project sqldelight by square.

the class PsiTokenSource method nextToken.

/* Colin: "the parsing lexer still has to return tokens that completely
	 cover the file (i.e. no gaps). This is one of the most significant
	 differences from a traditional compiler parser/lexer."
	  after lots of trial and error I finally just put the BAD_TOKEN
	  into the white space class so that they do not come to the parser
	  but that IDEA still knows about them.
	  parrt: this seems no longer to be true after Sam's re-factoring
	 */
@Override
public Token nextToken() {
    TokenElementType ideaTType = (TokenElementType) builder.getTokenType();
    int type;
    if (ideaTType == null) {
        type = Token.EOF;
    } else {
        type = ideaTType.getType();
    }
    int channel = Token.DEFAULT_CHANNEL;
    Pair<TokenSource, CharStream> source = new Pair<TokenSource, CharStream>(this, null);
    String text = builder.getTokenText();
    int start = builder.getCurrentOffset();
    int length = text != null ? text.length() : 0;
    int stop = start + length - 1;
    // PsiBuilder doesn't provide line, column info
    int line = 0;
    int charPositionInLine = 0;
    Token t = factory.create(source, type, text, channel, start, stop, line, charPositionInLine);
    builder.advanceLexer();
    //		System.out.println("TOKEN: "+t);
    return t;
}