Examples with TokenStream org.antlr.v4.runtime.TokenStream used on opensource projects

Example 1 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project crate by crate.

the class SqlParser method invokeParser.

private Node invokeParser(String name, String sql, Function<SqlBaseParser, ParserRuleContext> parseFunction) {
    try {
        SqlBaseLexer lexer = new SqlBaseLexer(new CaseInsensitiveStream(new ANTLRInputStream(sql)));
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        SqlBaseParser parser = new SqlBaseParser(tokenStream);
        parser.addParseListener(new PostProcessor());
        lexer.removeErrorListeners();
        lexer.addErrorListener(ERROR_LISTENER);
        parser.removeErrorListeners();
        parser.addErrorListener(ERROR_LISTENER);
        ParserRuleContext tree;
        try {
            // first, try parsing with potentially faster SLL mode
            parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            tree = parseFunction.apply(parser);
        } catch (ParseCancellationException ex) {
            // if we fail, parse with LL mode
            // rewind input stream
            tokenStream.reset();
            parser.reset();
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
            tree = parseFunction.apply(parser);
        }
        return new AstBuilder().visit(tree);
    } catch (StackOverflowError e) {
        throw new ParsingException(name + " is too large (stack overflow while parsing)");
    }
}

Example 2 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project pinot by linkedin.

the class Pql2Compiler method compileToBrokerRequest.

@Override
public BrokerRequest compileToBrokerRequest(String expression) throws Pql2CompilationException {
    try {
        //
        CharStream charStream = new ANTLRInputStream(expression);
        PQL2Lexer lexer = new PQL2Lexer(charStream);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
        PQL2Parser parser = new PQL2Parser(tokenStream);
        parser.setErrorHandler(new BailErrorStrategy());
        // Parse
        ParseTree parseTree = parser.root();
        ParseTreeWalker walker = new ParseTreeWalker();
        Pql2AstListener listener = new Pql2AstListener(expression);
        walker.walk(listener, parseTree);
        AstNode rootNode = listener.getRootNode();
        BrokerRequest brokerRequest = new BrokerRequest();
        rootNode.updateBrokerRequest(brokerRequest);
        return brokerRequest;
    } catch (Pql2CompilationException e) {
        throw e;
    } catch (Exception e) {
        throw new Pql2CompilationException(e.getMessage());
    }
}

Example 3 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project presto by prestodb.

the class SqlParser method invokeParser.

private Node invokeParser(String name, String sql, Function<SqlBaseParser, ParserRuleContext> parseFunction) {
    try {
        SqlBaseLexer lexer = new SqlBaseLexer(new CaseInsensitiveStream(new ANTLRInputStream(sql)));
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        SqlBaseParser parser = new SqlBaseParser(tokenStream);
        parser.addParseListener(new PostProcessor());
        lexer.removeErrorListeners();
        lexer.addErrorListener(ERROR_LISTENER);
        parser.removeErrorListeners();
        parser.addErrorListener(ERROR_LISTENER);
        ParserRuleContext tree;
        try {
            // first, try parsing with potentially faster SLL mode
            parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            tree = parseFunction.apply(parser);
        } catch (ParseCancellationException ex) {
            // if we fail, parse with LL mode
            // rewind input stream
            tokenStream.reset();
            parser.reset();
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
            tree = parseFunction.apply(parser);
        }
        return new AstBuilder().visit(tree);
    } catch (StackOverflowError e) {
        throw new ParsingException(name + " is too large (stack overflow while parsing)");
    }
}

Example 4 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class XPath method split.

// TODO: check for invalid token/rule names, bad syntax
public XPathElement[] split(String path) {
    ANTLRInputStream in;
    try {
        in = new ANTLRInputStream(new StringReader(path));
    } catch (IOException ioe) {
        throw new IllegalArgumentException("Could not read path: " + path, ioe);
    }
    XPathLexer lexer = new XPathLexer(in) {

        @Override
        public void recover(LexerNoViableAltException e) {
            throw e;
        }
    };
    lexer.removeErrorListeners();
    lexer.addErrorListener(new XPathLexerErrorListener());
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    try {
        tokenStream.fill();
    } catch (LexerNoViableAltException e) {
        int pos = lexer.getCharPositionInLine();
        String msg = "Invalid tokens or characters at index " + pos + " in path '" + path + "'";
        throw new IllegalArgumentException(msg, e);
    }
    List<Token> tokens = tokenStream.getTokens();
    //		System.out.println("path="+path+"=>"+tokens);
    List<XPathElement> elements = new ArrayList<XPathElement>();
    int n = tokens.size();
    int i = 0;
    loop: while (i < n) {
        Token el = tokens.get(i);
        Token next = null;
        switch(el.getType()) {
            case XPathLexer.ROOT:
            case XPathLexer.ANYWHERE:
                boolean anywhere = el.getType() == XPathLexer.ANYWHERE;
                i++;
                next = tokens.get(i);
                boolean invert = next.getType() == XPathLexer.BANG;
                if (invert) {
                    i++;
                    next = tokens.get(i);
                }
                XPathElement pathElement = getXPathElement(next, anywhere);
                pathElement.invert = invert;
                elements.add(pathElement);
                i++;
                break;
            case XPathLexer.TOKEN_REF:
            case XPathLexer.RULE_REF:
            case XPathLexer.WILDCARD:
                elements.add(getXPathElement(el, false));
                i++;
                break;
            case Token.EOF:
                break loop;
            default:
                throw new IllegalArgumentException("Unknowth path element " + el);
        }
    }
    return elements.toArray(new XPathElement[0]);
}

Example 5 with TokenStream

use of org.antlr.v4.runtime.TokenStream in project antlr4 by antlr.

the class DefaultErrorStrategy method sync.

/**
	 * The default implementation of {@link ANTLRErrorStrategy#sync} makes sure
	 * that the current lookahead symbol is consistent with what were expecting
	 * at this point in the ATN. You can call this anytime but ANTLR only
	 * generates code to check before subrules/loops and each iteration.
	 *
	 * <p>Implements Jim Idle's magic sync mechanism in closures and optional
	 * subrules. E.g.,</p>
	 *
	 * <pre>
	 * a : sync ( stuff sync )* ;
	 * sync : {consume to what can follow sync} ;
	 * </pre>
	 *
	 * At the start of a sub rule upon error, {@link #sync} performs single
	 * token deletion, if possible. If it can't do that, it bails on the current
	 * rule and uses the default error recovery, which consumes until the
	 * resynchronization set of the current rule.
	 *
	 * <p>If the sub rule is optional ({@code (...)?}, {@code (...)*}, or block
	 * with an empty alternative), then the expected set includes what follows
	 * the subrule.</p>
	 *
	 * <p>During loop iteration, it consumes until it sees a token that can start a
	 * sub rule or what follows loop. Yes, that is pretty aggressive. We opt to
	 * stay in the loop as long as possible.</p>
	 *
	 * <p><strong>ORIGINS</strong></p>
	 *
	 * <p>Previous versions of ANTLR did a poor job of their recovery within loops.
	 * A single mismatch token or missing token would force the parser to bail
	 * out of the entire rules surrounding the loop. So, for rule</p>
	 *
	 * <pre>
	 * classDef : 'class' ID '{' member* '}'
	 * </pre>
	 *
	 * input with an extra token between members would force the parser to
	 * consume until it found the next class definition rather than the next
	 * member definition of the current class.
	 *
	 * <p>This functionality cost a little bit of effort because the parser has to
	 * compare token set at the start of the loop and at each iteration. If for
	 * some reason speed is suffering for you, you can turn off this
	 * functionality by simply overriding this method as a blank { }.</p>
	 */
@Override
public void sync(Parser recognizer) throws RecognitionException {
    ATNState s = recognizer.getInterpreter().atn.states.get(recognizer.getState());
    // If already recovering, don't try to sync
    if (inErrorRecoveryMode(recognizer)) {
        return;
    }
    TokenStream tokens = recognizer.getInputStream();
    int la = tokens.LA(1);
    // try cheaper subset first; might get lucky. seems to shave a wee bit off
    IntervalSet nextTokens = recognizer.getATN().nextTokens(s);
    if (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la)) {
        return;
    }
    switch(s.getStateType()) {
        case ATNState.BLOCK_START:
        case ATNState.STAR_BLOCK_START:
        case ATNState.PLUS_BLOCK_START:
        case ATNState.STAR_LOOP_ENTRY:
            // report error and recover if possible
            if (singleTokenDeletion(recognizer) != null) {
                return;
            }
            throw new InputMismatchException(recognizer);
        case ATNState.PLUS_LOOP_BACK:
        case ATNState.STAR_LOOP_BACK:
            //			System.err.println("at loop back: "+s.getClass().getSimpleName());
            reportUnwantedToken(recognizer);
            IntervalSet expecting = recognizer.getExpectedTokens();
            IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));
            consumeUntil(recognizer, whatFollowsLoopIterationOrRule);
            break;
        default:
            // do nothing if we can't identify the exact kind of ATN state
            break;
    }
}