Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 1 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project elasticsearch by elastic.

the class Walker method buildAntlrTree.

private SourceContext buildAntlrTree(String source) {
    ANTLRInputStream stream = new ANTLRInputStream(source);
    PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName);
    PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
    ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName);
    lexer.removeErrorListeners();
    parser.removeErrorListeners();
    if (settings.isPicky()) {
        setupPicky(parser);
    }
    parser.setErrorHandler(strategy);
    return parser.source();
}

Example 2 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project claw-compiler by C2SM-RCM.

the class ClawLanguage method analyze.

/**
   * Analyze a raw string input and match it with the CLAW language definition.
   *
   * @param rawPragma A raw pragma statement to be analyzed against the CLAW
   *                  language.
   * @param lineno    Line number of the pragma statement.
   * @param generator Accelerator directive generator.
   * @param target    Target that influences the code transformation.
   * @return A ClawLanguage object with the corresponding extracted information.
   * @throws IllegalDirectiveException If directive does not follow the CLAW
   *                                   language specification.
   */
private static ClawLanguage analyze(String rawPragma, int lineno, AcceleratorGenerator generator, Target target) throws IllegalDirectiveException {
    // Remove additional claw keyword
    rawPragma = nakenize(rawPragma);
    // Discard the ignored code after the claw ignore directive
    if (rawPragma.toLowerCase().contains(IGNORE)) {
        rawPragma = rawPragma.substring(0, rawPragma.toLowerCase().indexOf(IGNORE) + IGNORE.length());
    }
    // Instantiate the lexer with the raw string input
    ClawLexer lexer = new ClawLexer(CharStreams.fromString(rawPragma));
    // Get a list of matched tokens
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // Pass the tokens to the parser
    ClawParser parser = new ClawParser(tokens);
    parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    try {
        // Start the parser analysis from the "analyze" entry point
        ClawParser.AnalyzeContext ctx = parser.analyze();
        // Get the ClawLanguage object return by the parser after analysis.
        ctx.l.setAcceleratorGenerator(generator);
        ctx.l.setTarget(target);
        return ctx.l;
    } catch (ParseCancellationException pcex) {
        if (pcex.getCause() instanceof InputMismatchException) {
            InputMismatchException imex = (InputMismatchException) pcex.getCause();
            throw new IllegalDirectiveException(getTokens(imex.getExpectedTokens(), parser), lineno, imex.getOffendingToken().getCharPositionInLine());
        } else if (pcex.getCause() instanceof NoViableAltException) {
            NoViableAltException nvex = (NoViableAltException) pcex.getCause();
            throw new IllegalDirectiveException(nvex.getOffendingToken(), getTokens(nvex.getExpectedTokens(), parser), lineno, nvex.getOffendingToken().getCharPositionInLine());
        }
        throw new IllegalDirectiveException(rawPragma, "Unsupported construct", lineno, 0);
    }
}

Example 3 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project aic-praise by aic-sri-international.

the class HOGMCodeArea method computeHighlighting.

private static StyleSpans<Collection<String>> computeHighlighting(String text) {
    StyleSpansBuilder<Collection<String>> spansBuilder = new StyleSpansBuilder<>();
    int lastTokenEnd = 0;
    ANTLRInputStream input = new ANTLRInputStream(text);
    HOGMLexer lexer = new HOGMLexer(input);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    for (int i = 0; i < tokens.size(); i++) {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF) {
            break;
        }
        String styleClass;
        if (t.getType() == HOGMLexer.COMMENT || t.getType() == HOGMLexer.LINE_COMMENT) {
            styleClass = "hogmCodeComment";
        } else if (HOGMTerminalSymbols.isTerminalSymbol(t.getText())) {
            styleClass = "hogmCodeKeyword";
        } else {
            styleClass = "hogmCodeOther";
        }
        int spacing = t.getStartIndex() - lastTokenEnd;
        if (spacing > 0) {
            spansBuilder.add(Collections.emptyList(), spacing);
        }
        int stylesize = (t.getStopIndex() - t.getStartIndex()) + 1;
        spansBuilder.add(Collections.singleton(styleClass), stylesize);
        lastTokenEnd = t.getStopIndex() + 1;
    }
    return spansBuilder.create();
}

Example 4 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project presto by prestodb.

the class SqlParser method invokeParser.

private Node invokeParser(String name, String sql, Function<SqlBaseParser, ParserRuleContext> parseFunction) {
    try {
        SqlBaseLexer lexer = new SqlBaseLexer(new CaseInsensitiveStream(new ANTLRInputStream(sql)));
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        SqlBaseParser parser = new SqlBaseParser(tokenStream);
        parser.addParseListener(new PostProcessor());
        lexer.removeErrorListeners();
        lexer.addErrorListener(ERROR_LISTENER);
        parser.removeErrorListeners();
        parser.addErrorListener(ERROR_LISTENER);
        ParserRuleContext tree;
        try {
            // first, try parsing with potentially faster SLL mode
            parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            tree = parseFunction.apply(parser);
        } catch (ParseCancellationException ex) {
            // if we fail, parse with LL mode
            // rewind input stream
            tokenStream.reset();
            parser.reset();
            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
            tree = parseFunction.apply(parser);
        }
        return new AstBuilder().visit(tree);
    } catch (StackOverflowError e) {
        throw new ParsingException(name + " is too large (stack overflow while parsing)");
    }
}

Example 5 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class BaseBrowserTest method writeLexerTestFile.

protected void writeLexerTestFile(String lexerName, boolean showDFA) {
    String html = "<!DOCTYPE html>\r\n" + "<html>\r\n" + "	<head>\r\n" + "		<script src='lib/require.js'></script>\r\n" + "		<script>\r\n" + "			antlr4 = null;\r\n" + "			listener = null;\r\n" + "			" + lexerName + " = null;\r\n" + "\r\n" + "			loadLexer = function() {\r\n" + "				try {\r\n" + "					antlr4 = require('antlr4/index');\r\n" + "					" + lexerName + " = require('./parser/" + lexerName + "');\r\n" + "				} catch (ex) {\r\n" + "					document.getElementById('errors').value = ex.toString();\r\n" + "				}\r\n" + "				listener = function() {\r\n" + "					antlr4.error.ErrorListener.call(this);\r\n" + "					return this;\r\n" + "				}\r\n" + "				listener.prototype = Object.create(antlr4.error.ErrorListener.prototype);\r\n" + "				listener.prototype.constructor = listener;\r\n" + "				listener.prototype.syntaxError = function(recognizer, offendingSymbol, line, column, msg, e) {\r\n" + "    				document.getElementById('errors').value += 'line ' + line + ':' + column + ' ' + msg + '\\r\\n';\r\n" + "				};\r\n" + "			}\r\n" + "\r\n" + "			test = function() {\r\n" + "				document.getElementById('output').value = ''\r\n" + "				var input = document.getElementById('input').value;\r\n" + "			var chars = new antlr4.InputStream(input, true);\r\n" + "    			var lexer = new " + lexerName + "." + lexerName + "(chars);\r\n" + "				lexer._listeners = [new listener()];\r\n" + "    			var stream = new antlr4.CommonTokenStream(lexer);\r\n" + "    			stream.fill();\r\n" + "    			for(var i=0; i<stream.tokens.length; i++) {\r\n" + "					document.getElementById('output').value += stream.tokens[i].toString() + '\\r\\n';\r\n" + "    			}\n" + (showDFA ? "    			document.getElementById('output').value += lexer._interp.decisionToDFA[antlr4.Lexer.DEFAULT_MODE].toLexerString();\r\n" : "") + "			};\r\n" + "\r\n" + "		</script>\r\n" + "	</head>\r\n" + "	<body>\r\n" + "		<textarea id='input'></textarea><br>\r\n" + "		<button id='load' type='button' onclick='loadLexer()'>Load</button><br>\r\n" + "		<button id='submit' type='button' onclick='test()'>Test</button><br>\r\n" + "		<textarea id='output'></textarea><br>\r\n" + "		<textarea id='errors'></textarea><br>\r\n" + "	</body>\r\n" + "</html>\r\n";
    writeFile(httpdir, "Test.html", html);
}