Examples with TokenStream org.antlr.v4.runtime.TokenStream used on opensource projects

Example 16 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class Tool method parse.

public GrammarRootAST parse(String fileName, CharStream in) {
    try {
        GrammarASTAdaptor adaptor = new GrammarASTAdaptor(in);
        ToolANTLRLexer lexer = new ToolANTLRLexer(in, this);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        lexer.tokens = tokens;
        ToolANTLRParser p = new ToolANTLRParser(tokens, this);
        p.setTreeAdaptor(adaptor);
        try {
            ParserRuleReturnScope r = p.grammarSpec();
            GrammarAST root = (GrammarAST) r.getTree();
            if (root instanceof GrammarRootAST) {
                ((GrammarRootAST) root).hasErrors = lexer.getNumberOfSyntaxErrors() > 0 || p.getNumberOfSyntaxErrors() > 0;
                assert ((GrammarRootAST) root).tokenStream == tokens;
                if (grammarOptions != null) {
                    ((GrammarRootAST) root).cmdLineOptions = grammarOptions;
                }
                return ((GrammarRootAST) root);
            }
        } catch (v3TreeGrammarException e) {
            errMgr.grammarError(ErrorType.V3_TREE_GRAMMAR, fileName, e.location);
        }
        return null;
    } catch (RecognitionException re) {
        // TODO: do we gen errors now?
        ErrorManager.internalError("can't generate this message at moment; antlr recovers");
    }
    return null;
}

Example 17 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project pinot by linkedin.

the class Pql2Compiler method compileToExpressionTree.

@Override
public TransformExpressionTree compileToExpressionTree(String expression) {
    CharStream charStream = new ANTLRInputStream(expression);
    PQL2Lexer lexer = new PQL2Lexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
    PQL2Parser parser = new PQL2Parser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    // Parse
    ParseTree parseTree = parser.expression();
    ParseTreeWalker walker = new ParseTreeWalker();
    Pql2AstListener listener = new Pql2AstListener(expression);
    walker.walk(listener, parseTree);
    final AstNode rootNode = listener.getRootNode();
    return TransformExpressionTree.buildTree(rootNode);
}

Example 18 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestBufferedTokenStream method testCompleteBuffer.

@Test
public void testCompleteBuffer() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);
    int i = 1;
    Token t = tokens.LT(i);
    while (t.getType() != Token.EOF) {
        i++;
        t = tokens.LT(i);
    }
    // push it past end
    tokens.LT(i++);
    tokens.LT(i++);
    String result = tokens.getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);
}

Example 19 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestATNParserPrediction method checkDFAConstruction.

public void checkDFAConstruction(LexerGrammar lg, Grammar g, int decision, String[] inputString, String[] dfaString) {
    //		Tool.internalOption_ShowATNConfigsInDFA = true;
    ATN lexatn = createATN(lg, true);
    LexerATNSimulator lexInterp = new LexerATNSimulator(lexatn, new DFA[] { new DFA(lexatn.getDecisionState(Lexer.DEFAULT_MODE)) }, new PredictionContextCache());
    semanticProcess(lg);
    g.importVocab(lg);
    semanticProcess(g);
    ParserInterpreterForTesting interp = new ParserInterpreterForTesting(g, null);
    for (int i = 0; i < inputString.length; i++) {
        // Check DFA
        IntegerList types = getTokenTypesViaATN(inputString[i], lexInterp);
        //			System.out.println(types);
        TokenStream input = new IntTokenStream(types);
        try {
            interp.adaptivePredict(input, decision, ParserRuleContext.EMPTY);
        } catch (NoViableAltException nvae) {
            nvae.printStackTrace(System.err);
        }
        DFA dfa = interp.parser.decisionToDFA[decision];
        assertEquals(dfaString[i], dfa.toString(g.getVocabulary()));
    }
}

Example 20 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class TestTokenStream method testBufferedTokenStreamReuseAfterFill.

/**
	 * This is a targeted regression test for antlr/antlr4#1584 ({@link BufferedTokenStream} cannot be reused after EOF).
	 */
@Test
public void testBufferedTokenStreamReuseAfterFill() {
    CharStream firstInput = new ANTLRInputStream("A");
    BufferedTokenStream tokenStream = new BufferedTokenStream(new VisitorBasicLexer(firstInput));
    tokenStream.fill();
    Assert.assertEquals(2, tokenStream.size());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
    Assert.assertEquals(Token.EOF, tokenStream.get(1).getType());
    CharStream secondInput = new ANTLRInputStream("AA");
    tokenStream.setTokenSource(new VisitorBasicLexer(secondInput));
    tokenStream.fill();
    Assert.assertEquals(3, tokenStream.size());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(0).getType());
    Assert.assertEquals(VisitorBasicLexer.A, tokenStream.get(1).getType());
    Assert.assertEquals(Token.EOF, tokenStream.get(2).getType());
}