Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 21 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project ksql by confluentinc.

the class KsqlParser method getParseTree.

private ParserRuleContext getParseTree(String sql) {
    SqlBaseLexer sqlBaseLexer = new SqlBaseLexer(new CaseInsensitiveStream(new ANTLRInputStream(sql)));
    CommonTokenStream tokenStream = new CommonTokenStream(sqlBaseLexer);
    SqlBaseParser sqlBaseParser = new SqlBaseParser(tokenStream);
    sqlBaseLexer.removeErrorListeners();
    sqlBaseLexer.addErrorListener(ERROR_LISTENER);
    sqlBaseParser.removeErrorListeners();
    sqlBaseParser.addErrorListener(ERROR_LISTENER);
    Function<SqlBaseParser, ParserRuleContext> parseFunction = SqlBaseParser::statements;
    ParserRuleContext tree;
    try {
        // first, try parsing with potentially faster SLL mode
        sqlBaseParser.getInterpreter().setPredictionMode(PredictionMode.SLL);
        tree = parseFunction.apply(sqlBaseParser);
    } catch (ParseCancellationException ex) {
        // if we fail, parse with LL mode
        // rewind input stream
        tokenStream.reset();
        sqlBaseParser.reset();
        sqlBaseParser.getInterpreter().setPredictionMode(PredictionMode.LL);
        tree = parseFunction.apply(sqlBaseParser);
    }
    return tree;
}

Example 22 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project java by wavefrontHQ.

the class AbstractIngesterFormatter method getQueue.

protected Queue<Token> getQueue(String input) {
    DSWrapperLexer lexer = dsWrapperLexerThreadLocal.get();
    lexer.setInputStream(new ANTLRInputStream(input));
    CommonTokenStream commonTokenStream = new CommonTokenStream(lexer);
    commonTokenStream.fill();
    List<Token> tokens = commonTokenStream.getTokens();
    if (tokens.isEmpty()) {
        throw new RuntimeException("Could not parse: " + input);
    }
    // this is sensitive to the grammar in DSQuery.g4. We could just use the visitor but doing so
    // means we need to be creating the AST and instead we could just use the lexer. in any case,
    // we don't expect the graphite format to change anytime soon.
    // filter all EOF tokens first.
    Queue<Token> queue = tokens.stream().filter(t -> t.getType() != Lexer.EOF).collect(Collectors.toCollection(ArrayDeque::new));
    return queue;
}

Example 23 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project parseq by linkedin.

the class RequestConfigElement method parse.

static RequestConfigElement parse(String property, String key, Object value) throws RequestConfigKeyParsingException {
    RequestConfigKeyParsingErrorListener errorListener = new RequestConfigKeyParsingErrorListener();
    ANTLRInputStream input = new ANTLRInputStream(key);
    RequestConfigKeyLexer lexer = new RequestConfigKeyLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(errorListener);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    RequestConfigKeyParser parser = new RequestConfigKeyParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(errorListener);
    KeyContext keyTree = parser.key();
    if (!errorListener.hasErrors()) {
        InboundContext inbound = keyTree.inbound();
        OutboundContext outbound = keyTree.outbound();
        Optional<String> inboundName = handlingWildcard(inbound.restResource());
        Optional<String> outboundName = handlingWildcard(outbound.restResource());
        Optional<String> inboundOp = getOpIn(inbound.operationIn());
        Optional<ResourceMethod> outboundOp = getOpOut(outbound.operationOut());
        Optional<String> inboundOpName = inboundOp.flatMap(method -> getOpInName(method, inbound.operationIn()));
        Optional<String> outboundOpName = outboundOp.flatMap(method -> getOpOutName(method, outbound.operationOut()));
        return new RequestConfigElement(key, coerceValue(property, value), property, inboundName, outboundName, inboundOpName, outboundOpName, inboundOp, outboundOp);
    } else {
        throw new RequestConfigKeyParsingException("Error" + ((errorListener.errorsSize() > 1) ? "s" : "") + " parsing key: " + key + "\n" + errorListener);
    }
}

Example 24 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project siddhi by wso2.

the class SiddhiCompiler method parseQuery.

public static Query parseQuery(String query) throws SiddhiParserException {
    CharStream input = CharStreams.fromString(query);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.query_final();
    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (Query) eval.visit(tree);
}

Example 25 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project siddhi by wso2.

the class SiddhiCompiler method parseFunctionDefinition.

public static FunctionDefinition parseFunctionDefinition(String functionDefinition) throws SiddhiParserException {
    CharStream input = CharStreams.fromString(functionDefinition);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.definition_function_final();
    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (FunctionDefinition) eval.visit(tree);
}