Examples with Lexer org.antlr.v4.runtime.Lexer used on opensource projects

Example 1 with Lexer

use of org.antlr.v4.runtime.Lexer in project elasticsearch by elastic.

the class Walker method buildAntlrTree.

private SourceContext buildAntlrTree(String source) {
    ANTLRInputStream stream = new ANTLRInputStream(source);
    PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName);
    PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
    ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName);
    lexer.removeErrorListeners();
    parser.removeErrorListeners();
    if (settings.isPicky()) {
        setupPicky(parser);
    }
    parser.setErrorHandler(strategy);
    return parser.source();
}

Example 2 with Lexer

use of org.antlr.v4.runtime.Lexer in project pinot by linkedin.

the class Pql2Compiler method compileToBrokerRequest.

@Override
public BrokerRequest compileToBrokerRequest(String expression) throws Pql2CompilationException {
    try {
        //
        CharStream charStream = new ANTLRInputStream(expression);
        PQL2Lexer lexer = new PQL2Lexer(charStream);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
        PQL2Parser parser = new PQL2Parser(tokenStream);
        parser.setErrorHandler(new BailErrorStrategy());
        // Parse
        ParseTree parseTree = parser.root();
        ParseTreeWalker walker = new ParseTreeWalker();
        Pql2AstListener listener = new Pql2AstListener(expression);
        walker.walk(listener, parseTree);
        AstNode rootNode = listener.getRootNode();
        BrokerRequest brokerRequest = new BrokerRequest();
        rootNode.updateBrokerRequest(brokerRequest);
        return brokerRequest;
    } catch (Pql2CompilationException e) {
        throw e;
    } catch (Exception e) {
        throw new Pql2CompilationException(e.getMessage());
    }
}

Example 3 with Lexer

use of org.antlr.v4.runtime.Lexer in project claw-compiler by C2SM-RCM.

the class ClawLanguage method analyze.

/**
   * Analyze a raw string input and match it with the CLAW language definition.
   *
   * @param rawPragma A raw pragma statement to be analyzed against the CLAW
   *                  language.
   * @param lineno    Line number of the pragma statement.
   * @param generator Accelerator directive generator.
   * @param target    Target that influences the code transformation.
   * @return A ClawLanguage object with the corresponding extracted information.
   * @throws IllegalDirectiveException If directive does not follow the CLAW
   *                                   language specification.
   */
private static ClawLanguage analyze(String rawPragma, int lineno, AcceleratorGenerator generator, Target target) throws IllegalDirectiveException {
    // Remove additional claw keyword
    rawPragma = nakenize(rawPragma);
    // Discard the ignored code after the claw ignore directive
    if (rawPragma.toLowerCase().contains(IGNORE)) {
        rawPragma = rawPragma.substring(0, rawPragma.toLowerCase().indexOf(IGNORE) + IGNORE.length());
    }
    // Instantiate the lexer with the raw string input
    ClawLexer lexer = new ClawLexer(CharStreams.fromString(rawPragma));
    // Get a list of matched tokens
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // Pass the tokens to the parser
    ClawParser parser = new ClawParser(tokens);
    parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    try {
        // Start the parser analysis from the "analyze" entry point
        ClawParser.AnalyzeContext ctx = parser.analyze();
        // Get the ClawLanguage object return by the parser after analysis.
        ctx.l.setAcceleratorGenerator(generator);
        ctx.l.setTarget(target);
        return ctx.l;
    } catch (ParseCancellationException pcex) {
        if (pcex.getCause() instanceof InputMismatchException) {
            InputMismatchException imex = (InputMismatchException) pcex.getCause();
            throw new IllegalDirectiveException(getTokens(imex.getExpectedTokens(), parser), lineno, imex.getOffendingToken().getCharPositionInLine());
        } else if (pcex.getCause() instanceof NoViableAltException) {
            NoViableAltException nvex = (NoViableAltException) pcex.getCause();
            throw new IllegalDirectiveException(nvex.getOffendingToken(), getTokens(nvex.getExpectedTokens(), parser), lineno, nvex.getOffendingToken().getCharPositionInLine());
        }
        throw new IllegalDirectiveException(rawPragma, "Unsupported construct", lineno, 0);
    }
}

Example 4 with Lexer

use of org.antlr.v4.runtime.Lexer in project aic-praise by aic-sri-international.

the class HOGMCodeArea method computeHighlighting.

private static StyleSpans<Collection<String>> computeHighlighting(String text) {
    StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
    int lastTokenEnd = 0;
    ANTLRInputStream input = new ANTLRInputStream(text);
    HOGMLexer lexer = new HOGMLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF) {
            break;
        }
        String styleClass;
        if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
            styleClass = "hogmCodeComment";
        } else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
            styleClass = "hogmCodeKeyword";
        } else {
            styleClass = "hogmCodeOther";
        }
        int spacing = t.getStartIndex() - lastTokenEnd;
        if (spacing > 0) {
            spansBuilder.add(Collections.emptyList(), spacing);
        }
        int stylesize = (t.getStopIndex() - t.getStartIndex()) + 1;
        spansBuilder.add(Collections.singleton(styleClass), stylesize);
        lastTokenEnd = t.getStopIndex() + 1;
    }
    return spansBuilder.create();
}

Example 5 with Lexer

use of org.antlr.v4.runtime.Lexer in project presto by prestodb.

the class SqlParser method invokeParser.

private Node invokeParser(String name, String sql, Function<SqlBaseParser, ParserRuleContext> parseFunction) {
    try {
        SqlBaseLexer lexer = new SqlBaseLexer(new CaseInsensitiveStream(new ANTLRInputStream(sql)));
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        SqlBaseParser parser = new SqlBaseParser(tokenStream);
        parser.addParseListener(new PostProcessor());
        lexer.removeErrorListeners();
        lexer.addErrorListener(ERROR_LISTENER);
        parser.removeErrorListeners();
        parser.addErrorListener(ERROR_LISTENER);
        ParserRuleContext tree;
        try {
            // first, try parsing with potentially faster SLL mode
            parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            tree = parseFunction.apply(parser);
        } catch (ParseCancellationException ex) {
            // if we fail, parse with LL mode
            // rewind input stream
            tokenStream.reset();
            parser.reset();
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
            tree = parseFunction.apply(parser);
        }
        return new AstBuilder().visit(tree);
    } catch (StackOverflowError e) {
        throw new ParsingException(name + " is too large (stack overflow while parsing)");
    }
}