Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 56 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project druid by druid-io.

the class Parser method parse.

public static Expr parse(String in, boolean withFlatten) {
    ExprLexer lexer = new ExprLexer(new ANTLRInputStream(in));
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    ExprParser parser = new ExprParser(tokens);
    parser.setBuildParseTree(true);
    ParseTree parseTree = parser.expr();
    ParseTreeWalker walker = new ParseTreeWalker();
    ExprListenerImpl listener = new ExprListenerImpl(parseTree);
    walker.walk(listener, parseTree);
    return withFlatten ? flatten(listener.getAST()) : listener.getAST();
}

Example 57 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project rest.li by linkedin.

the class PdlSchemaParser method parse.

/**
   * Parse a JSON representation of a schema from a {{java.io.Reader}}.
   *
   * The top level {{DataSchema}}'s parsed are in {{#topLevelDataSchemas}}.
   * These are the types that are not defined within other types.
   * Parse errors are in {{#errorMessageBuilder}} and indicated
   * by {{#hasError()}}.
   *
   * @param reader with the JSON representation of the schema.
   */
public void parse(Reader reader) {
    try {
        ErrorRecorder errorRecorder = new ErrorRecorder();
        PdlLexer lexer;
        try {
            lexer = new PdlLexer(new ANTLRInputStream(reader));
        } catch (IOException e) {
            ParseError error = new ParseError(new ParseErrorLocation(0, 0), e.getMessage());
            startErrorMessage(error).append(error.message).append(NEWLINE);
            return;
        }
        lexer.removeErrorListeners();
        lexer.addErrorListener(errorRecorder);
        PdlParser parser = new PdlParser(new CommonTokenStream(lexer));
        parser.removeErrorListeners();
        parser.addErrorListener(errorRecorder);
        DocumentContext antlrDocument = parser.document();
        parse(antlrDocument);
        if (errorRecorder.errors.size() > 0) {
            for (ParseError error : errorRecorder.errors) {
                startErrorMessage(error).append(error.message).append(NEWLINE);
            }
        }
    } catch (ParseException e) {
        startErrorMessage(e.error).append(e.getMessage()).append(NEWLINE);
    } catch (Throwable t) {
        ParseError parseError = new ParseError(new ParseErrorLocation(0, 0), null);
        startErrorMessage(parseError).append("Unexpected parser error: ").append(ExceptionUtils.getStackTrace(t)).append(NEWLINE);
    }
}

Example 58 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class BaseJavaTest method getParserAndLexer.

public Pair<Parser, Lexer> getParserAndLexer(String input, String parserName, String lexerName) throws Exception {
    final Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(lexerName);
    final Class<? extends Parser> parserClass = loadParserClassFromTempDir(parserName);
    ANTLRInputStream in = new ANTLRInputStream(new StringReader(input));
    Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
    Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance(in);
    Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
    Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Parser parser = pctor.newInstance(tokens);
    return new Pair<Parser, Lexer>(parser, lexer);
}

Example 59 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class BaseCppTest method writeParserTestFile.

protected void writeParserTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug, boolean trace) {
    if (!parserStartRuleName.endsWith(")"))
        parserStartRuleName += "()";
    ST outputFileST = new ST("#include \\<iostream>\n" + "\n" + "#include \"antlr4-runtime.h\"\n" + "#include \"<lexerName>.h\"\n" + "#include \"<parserName>.h\"\n" + "\n" + "using namespace antlr4;\n" + "\n" + "class TreeShapeListener : public tree::ParseTreeListener {\n" + "public:\n" + "  void visitTerminal(tree::TerminalNode *) override {}\n" + "  void visitErrorNode(tree::ErrorNode *) override {}\n" + "  void exitEveryRule(ParserRuleContext *) override {}\n" + "  void enterEveryRule(ParserRuleContext *ctx) override {\n" + "    for (auto child : ctx->children) {\n" + "      tree::ParseTree *parent = child->parent;\n" + "      ParserRuleContext *rule = dynamic_cast\\<ParserRuleContext *>(parent);\n" + "      if (rule != ctx) {\n" + "        throw \"Invalid parse tree shape detected.\";\n" + "      }\n" + "    }\n" + "  }\n" + "};\n" + "\n" + "\n" + "int main(int argc, const char* argv[]) {\n" + "  ANTLRFileStream input(argv[1]);\n" + "  <lexerName> lexer(&input);\n" + "  CommonTokenStream tokens(&lexer);\n" + "<createParser>" + "\n" + "  tree::ParseTree *tree = parser.<parserStartRuleName>;\n" + "  TreeShapeListener listener;\n" + "  tree::ParseTreeWalker::DEFAULT.walk(&listener, tree);\n" + "\n" + "  return 0;\n" + "}\n");
    String stSource = "  <parserName> parser(&tokens);\n";
    if (debug) {
        stSource += "  DiagnosticErrorListener errorListener;\n";
        stSource += "  parser.addErrorListener(&errorListener);\n";
    }
    if (trace)
        stSource += "  parser.setTrace(true);\n";
    ST createParserST = new ST(stSource);
    outputFileST.add("createParser", createParserST);
    outputFileST.add("parserName", parserName);
    outputFileST.add("lexerName", lexerName);
    outputFileST.add("listenerName", listenerName);
    outputFileST.add("visitorName", visitorName);
    outputFileST.add("parserStartRuleName", parserStartRuleName);
    writeFile(tmpdir, "Test.cpp", outputFileST.render());
}

Example 60 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestAmbigParseTrees method testInterpAtSpecificAlt.

void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g, String startRule, int startAlt, String input, String expectedParseTree) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
    Transition tr = ruleStartState.transition(0);
    ATNState t2 = tr.target;
    if (!(t2 instanceof BasicBlockStartState)) {
        throw new IllegalArgumentException("rule has no decision: " + startRule);
    }
    parser.addDecisionOverride(((DecisionState) t2).decision, 0, startAlt);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
}