Examples with CommonTokenStream org.antlr.v4.runtime.CommonTokenStream used on opensource projects

Example 91 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestTokenStreamRewriter method testCombine3Inserts.

@Test
public void testCombine3Inserts() throws Exception {
    LexerGrammar g = new LexerGrammar("lexer grammar T;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n");
    String input = "abc";
    LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream stream = new CommonTokenStream(lexEngine);
    stream.fill();
    TokenStreamRewriter tokens = new TokenStreamRewriter(stream);
    tokens.insertBefore(1, "x");
    tokens.insertBefore(0, "y");
    tokens.insertBefore(1, "z");
    String result = tokens.getText();
    String expecting = "yazxbc";
    assertEquals(expecting, result);
}

Example 92 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class BaseJavaTest method getParserAndLexer.

public Pair<Parser, Lexer> getParserAndLexer(String input, String parserName, String lexerName) throws Exception {
    final Class<? extends Lexer> lexerClass = loadLexerClassFromTempDir(lexerName);
    final Class<? extends Parser> parserClass = loadParserClassFromTempDir(parserName);
    ANTLRInputStream in = new ANTLRInputStream(new StringReader(input));
    Class<? extends Lexer> c = lexerClass.asSubclass(Lexer.class);
    Constructor<? extends Lexer> ctor = c.getConstructor(CharStream.class);
    Lexer lexer = ctor.newInstance(in);
    Class<? extends Parser> pc = parserClass.asSubclass(Parser.class);
    Constructor<? extends Parser> pctor = pc.getConstructor(TokenStream.class);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    Parser parser = pctor.newInstance(tokens);
    return new Pair<Parser, Lexer>(parser, lexer);
}

Example 93 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class BaseCppTest method writeParserTestFile.

protected void writeParserTestFile(String parserName, String lexerName, String listenerName, String visitorName, String parserStartRuleName, boolean debug, boolean trace) {
    if (!parserStartRuleName.endsWith(")"))
        parserStartRuleName += "()";
    ST outputFileST = new ST("#include \\<iostream>\n" + "\n" + "#include \"antlr4-runtime.h\"\n" + "#include \"<lexerName>.h\"\n" + "#include \"<parserName>.h\"\n" + "\n" + "using namespace antlr4;\n" + "\n" + "class TreeShapeListener : public tree::ParseTreeListener {\n" + "public:\n" + "  void visitTerminal(tree::TerminalNode *) override {}\n" + "  void visitErrorNode(tree::ErrorNode *) override {}\n" + "  void exitEveryRule(ParserRuleContext *) override {}\n" + "  void enterEveryRule(ParserRuleContext *ctx) override {\n" + "    for (auto child : ctx->children) {\n" + "      tree::ParseTree *parent = child->parent;\n" + "      ParserRuleContext *rule = dynamic_cast\\<ParserRuleContext *>(parent);\n" + "      if (rule != ctx) {\n" + "        throw \"Invalid parse tree shape detected.\";\n" + "      }\n" + "    }\n" + "  }\n" + "};\n" + "\n" + "\n" + "int main(int argc, const char* argv[]) {\n" + "  ANTLRFileStream input(argv[1]);\n" + "  <lexerName> lexer(&input);\n" + "  CommonTokenStream tokens(&lexer);\n" + "<createParser>" + "\n" + "  tree::ParseTree *tree = parser.<parserStartRuleName>;\n" + "  TreeShapeListener listener;\n" + "  tree::ParseTreeWalker::DEFAULT.walk(&listener, tree);\n" + "\n" + "  return 0;\n" + "}\n");
    String stSource = "  <parserName> parser(&tokens);\n";
    if (debug) {
        stSource += "  DiagnosticErrorListener errorListener;\n";
        stSource += "  parser.addErrorListener(&errorListener);\n";
    }
    if (trace)
        stSource += "  parser.setTrace(true);\n";
    ST createParserST = new ST(stSource);
    outputFileST.add("createParser", createParserST);
    outputFileST.add("parserName", parserName);
    outputFileST.add("lexerName", lexerName);
    outputFileST.add("listenerName", listenerName);
    outputFileST.add("visitorName", visitorName);
    outputFileST.add("parserStartRuleName", parserStartRuleName);
    writeFile(tmpdir, "Test.cpp", outputFileST.render());
}

Example 94 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestAmbigParseTrees method testInterpAtSpecificAlt.

void testInterpAtSpecificAlt(LexerGrammar lg, Grammar g, String startRule, int startAlt, String input, String expectedParseTree) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    ParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    RuleStartState ruleStartState = g.atn.ruleToStartState[g.getRule(startRule).index];
    Transition tr = ruleStartState.transition(0);
    ATNState t2 = tr.target;
    if (!(t2 instanceof BasicBlockStartState)) {
        throw new IllegalArgumentException("rule has no decision: " + startRule);
    }
    parser.addDecisionOverride(((DecisionState) t2).decision, 0, startAlt);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    assertEquals(expectedParseTree, Trees.toStringTree(t, nodeTextProvider));
}

Example 95 with CommonTokenStream

use of org.antlr.v4.runtime.CommonTokenStream in project antlr4 by antlr.

the class TestGrammarParserInterpreter method testInterp.

InterpreterRuleContext testInterp(LexerGrammar lg, Grammar g, String startRule, String input, String expectedParseTree) {
    LexerInterpreter lexEngine = lg.createLexerInterpreter(new ANTLRInputStream(input));
    CommonTokenStream tokens = new CommonTokenStream(lexEngine);
    GrammarParserInterpreter parser = g.createGrammarParserInterpreter(tokens);
    ParseTree t = parser.parse(g.rules.get(startRule).index);
    InterpreterTreeTextProvider nodeTextProvider = new InterpreterTreeTextProvider(g.getRuleNames());
    String treeStr = Trees.toStringTree(t, nodeTextProvider);
    //		System.out.println("parse tree: "+treeStr);
    assertEquals(expectedParseTree, treeStr);
    return (InterpreterRuleContext) t;
}