Examples with Parser org.antlr.v4.runtime.Parser used on opensource projects

Example 81 with Parser

use of org.antlr.v4.runtime.Parser in project antlr4 by antlr.

the class GrammarTransformPipeline method extractImplicitLexer.

/** Build lexer grammar from combined grammar that looks like:
	 *
	 *  (COMBINED_GRAMMAR A
	 *      (tokens { X (= Y 'y'))
	 *      (OPTIONS (= x 'y'))
	 *      (@ members {foo})
	 *      (@ lexer header {package jj;})
	 *      (RULES (RULE .+)))
	 *
	 *  Move rules and actions to new tree, don't dup. Split AST apart.
	 *  We'll have this Grammar share token symbols later; don't generate
	 *  tokenVocab or tokens{} section.  Copy over named actions.
	 *
	 *  Side-effects: it removes children from GRAMMAR & RULES nodes
	 *                in combined AST.  Anything cut out is dup'd before
	 *                adding to lexer to avoid "who's ur daddy" issues
	 */
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
    GrammarRootAST combinedAST = combinedGrammar.ast;
    //tool.log("grammar", "before="+combinedAST.toStringTree());
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
    GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
    // MAKE A GRAMMAR ROOT and ID
    String lexerName = combinedAST.getChild(0).getText() + "Lexer";
    GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
    lexerAST.grammarType = ANTLRParser.LEXER;
    lexerAST.token.setInputStream(combinedAST.token.getInputStream());
    lexerAST.addChild((GrammarAST) adaptor.create(ANTLRParser.ID, lexerName));
    // COPY OPTIONS
    GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
    if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
        GrammarAST lexerOptionsRoot = (GrammarAST) adaptor.dupNode(optionsRoot);
        lexerAST.addChild(lexerOptionsRoot);
        GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
        for (GrammarAST o : options) {
            String optionName = o.getChild(0).getText();
            if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
                GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
                lexerOptionsRoot.addChild(optionTree);
                lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
            }
        }
    }
    // COPY all named actions, but only move those with lexer:: scope
    List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
    for (GrammarAST e : elements) {
        if (e.getType() == ANTLRParser.AT) {
            lexerAST.addChild((Tree) adaptor.dupTree(e));
            if (e.getChild(0).getText().equals("lexer")) {
                actionsWeMoved.add(e);
            }
        }
    }
    for (GrammarAST r : actionsWeMoved) {
        combinedAST.deleteChild(r);
    }
    GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
    if (combinedRulesRoot == null)
        return lexerAST;
    // MOVE lexer rules
    GrammarAST lexerRulesRoot = (GrammarAST) adaptor.create(ANTLRParser.RULES, "RULES");
    lexerAST.addChild(lexerRulesRoot);
    List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
    GrammarASTWithOptions[] rules;
    if (combinedRulesRoot.getChildCount() > 0) {
        rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
    } else {
        rules = new GrammarASTWithOptions[0];
    }
    for (GrammarASTWithOptions r : rules) {
        String ruleName = r.getChild(0).getText();
        if (Grammar.isTokenName(ruleName)) {
            lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
            rulesWeMoved.add(r);
        }
    }
    for (GrammarAST r : rulesWeMoved) {
        combinedRulesRoot.deleteChild(r);
    }
    // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
    List<Pair<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
    Set<String> stringLiterals = combinedGrammar.getStringLiterals();
    // add strings from combined grammar (and imported grammars) into lexer
    // put them first as they are keywords; must resolve ambigs to these rules
    //		tool.log("grammar", "strings from parser: "+stringLiterals);
    int insertIndex = 0;
    nextLit: for (String lit : stringLiterals) {
        // if lexer already has a rule for literal, continue
        if (litAliases != null) {
            for (Pair<GrammarAST, GrammarAST> pair : litAliases) {
                GrammarAST litAST = pair.b;
                if (lit.equals(litAST.getText()))
                    continue nextLit;
            }
        }
        // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
        String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
        // can't use wizard; need special node types
        GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
        BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
        AltAST alt = new AltAST(ANTLRParser.ALT);
        TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
        alt.addChild(slit);
        blk.addChild(alt);
        CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
        litRule.addChild(new TerminalAST(idToken));
        litRule.addChild(blk);
        lexerRulesRoot.insertChild(insertIndex, litRule);
        //			lexerRulesRoot.getChildren().add(0, litRule);
        // reset indexes and set litRule parent
        lexerRulesRoot.freshenParentAndChildIndexes();
        // next literal will be added after the one just added
        insertIndex++;
    }
    // TODO: take out after stable if slow
    lexerAST.sanityCheckParentAndChildIndexes();
    combinedAST.sanityCheckParentAndChildIndexes();
    //		tool.log("grammar", combinedAST.toTokenString());
    combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
    combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
    if (lexerRulesRoot.getChildCount() == 0)
        return null;
    return lexerAST;
}

Example 82 with Parser

use of org.antlr.v4.runtime.Parser in project antlr4 by antlr.

the class TestVisitors method testVisitErrorNode.

/**
	 * This test verifies the basic behavior of visitors, with an emphasis on
	 * {@link AbstractParseTreeVisitor#visitErrorNode}.
	 */
@Test
public void testVisitErrorNode() {
    String input = "";
    VisitorBasicLexer lexer = new VisitorBasicLexer(new ANTLRInputStream(input));
    VisitorBasicParser parser = new VisitorBasicParser(new CommonTokenStream(lexer));
    final List<String> errors = new ArrayList<>();
    parser.removeErrorListeners();
    parser.addErrorListener(new BaseErrorListener() {

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            errors.add("line " + line + ":" + charPositionInLine + " " + msg);
        }
    });
    VisitorBasicParser.SContext context = parser.s();
    Assert.assertEquals("(s <missing 'A'> <EOF>)", context.toStringTree(parser));
    Assert.assertEquals(1, errors.size());
    Assert.assertEquals("line 1:0 missing 'A' at '<EOF>'", errors.get(0));
    VisitorBasicVisitor<String> listener = new VisitorBasicBaseVisitor<String>() {

        @Override
        public String visitErrorNode(ErrorNode node) {
            return "Error encountered: " + node.getSymbol();
        }

        @Override
        protected String defaultResult() {
            return "";
        }

        @Override
        protected String aggregateResult(String aggregate, String nextResult) {
            return aggregate + nextResult;
        }
    };
    String result = listener.visit(context);
    String expected = "Error encountered: [@-1,-1:-1='<missing 'A'>',<1>,1:0]";
    Assert.assertEquals(expected, result);
}

Example 83 with Parser

use of org.antlr.v4.runtime.Parser in project antlr4 by antlr.

the class TestVisitors method testCalculatorVisitor.

/**
	 * This test verifies that the visitor correctly dispatches calls for labeled outer alternatives.
	 */
@Test
public void testCalculatorVisitor() {
    String input = "2 + 8 / 2";
    VisitorCalcLexer lexer = new VisitorCalcLexer(new ANTLRInputStream(input));
    VisitorCalcParser parser = new VisitorCalcParser(new CommonTokenStream(lexer));
    VisitorCalcParser.SContext context = parser.s();
    Assert.assertEquals("(s (expr (expr 2) + (expr (expr 8) / (expr 2))) <EOF>)", context.toStringTree(parser));
    VisitorCalcVisitor<Integer> listener = new VisitorCalcBaseVisitor<Integer>() {

        @Override
        public Integer visitS(VisitorCalcParser.SContext ctx) {
            return visit(ctx.expr());
        }

        @Override
        public Integer visitNumber(VisitorCalcParser.NumberContext ctx) {
            return Integer.valueOf(ctx.INT().getText());
        }

        @Override
        public Integer visitMultiply(VisitorCalcParser.MultiplyContext ctx) {
            Integer left = visit(ctx.expr(0));
            Integer right = visit(ctx.expr(1));
            if (ctx.MUL() != null) {
                return left * right;
            } else {
                return left / right;
            }
        }

        @Override
        public Integer visitAdd(VisitorCalcParser.AddContext ctx) {
            Integer left = visit(ctx.expr(0));
            Integer right = visit(ctx.expr(1));
            if (ctx.ADD() != null) {
                return left + right;
            } else {
                return left - right;
            }
        }

        @Override
        protected Integer defaultResult() {
            throw new RuntimeException("Should not be reachable");
        }

        @Override
        protected Integer aggregateResult(Integer aggregate, Integer nextResult) {
            throw new RuntimeException("Should not be reachable");
        }
    };
    int result = listener.visit(context);
    int expected = 6;
    Assert.assertEquals(expected, result);
}

Example 84 with Parser

use of org.antlr.v4.runtime.Parser in project antlr4 by antlr.

the class BaseBrowserTest method writeLexerTestFile.

protected void writeLexerTestFile(String lexerName, boolean showDFA) {
    String html = "<!DOCTYPE html>\r\n" + "<html>\r\n" + "	<head>\r\n" + "		<script src='lib/require.js'></script>\r\n" + "		<script>\r\n" + "			antlr4 = null;\r\n" + "			listener = null;\r\n" + "			" + lexerName + " = null;\r\n" + "\r\n" + "			loadLexer = function() {\r\n" + "				try {\r\n" + "					antlr4 = require('antlr4/index');\r\n" + "					" + lexerName + " = require('./parser/" + lexerName + "');\r\n" + "				} catch (ex) {\r\n" + "					document.getElementById('errors').value = ex.toString();\r\n" + "				}\r\n" + "				listener = function() {\r\n" + "					antlr4.error.ErrorListener.call(this);\r\n" + "					return this;\r\n" + "				}\r\n" + "				listener.prototype = Object.create(antlr4.error.ErrorListener.prototype);\r\n" + "				listener.prototype.constructor = listener;\r\n" + "				listener.prototype.syntaxError = function(recognizer, offendingSymbol, line, column, msg, e) {\r\n" + "    				document.getElementById('errors').value += 'line ' + line + ':' + column + ' ' + msg + '\\r\\n';\r\n" + "				};\r\n" + "			}\r\n" + "\r\n" + "			test = function() {\r\n" + "				document.getElementById('output').value = ''\r\n" + "				var input = document.getElementById('input').value;\r\n" + "			var chars = new antlr4.InputStream(input, true);\r\n" + "    			var lexer = new " + lexerName + "." + lexerName + "(chars);\r\n" + "				lexer._listeners = [new listener()];\r\n" + "    			var stream = new antlr4.CommonTokenStream(lexer);\r\n" + "    			stream.fill();\r\n" + "    			for(var i=0; i<stream.tokens.length; i++) {\r\n" + "					document.getElementById('output').value += stream.tokens[i].toString() + '\\r\\n';\r\n" + "    			}\n" + (showDFA ? "    			document.getElementById('output').value += lexer._interp.decisionToDFA[antlr4.Lexer.DEFAULT_MODE].toLexerString();\r\n" : "") + "			};\r\n" + "\r\n" + "		</script>\r\n" + "	</head>\r\n" + "	<body>\r\n" + "		<textarea id='input'></textarea><br>\r\n" + "		<button id='load' type='button' onclick='loadLexer()'>Load</button><br>\r\n" + "		<button id='submit' type='button' onclick='test()'>Test</button><br>\r\n" + "		<textarea id='output'></textarea><br>\r\n" + "		<textarea id='errors'></textarea><br>\r\n" + "	</body>\r\n" + "</html>\r\n";
    writeFile(httpdir, "Test.html", html);
}

Example 85 with Parser

use of org.antlr.v4.runtime.Parser in project antlr4 by antlr.

the class BasePythonTest method execStartRule.

public ParseTree execStartRule(String startRuleName, Parser parser) throws IllegalAccessException, InvocationTargetException, NoSuchMethodException {
    Method startRule = null;
    Object[] args = null;
    try {
        startRule = parser.getClass().getMethod(startRuleName);
    } catch (NoSuchMethodException nsme) {
        // try with int _p arg for recursive func
        startRule = parser.getClass().getMethod(startRuleName, int.class);
        args = new Integer[] { 0 };
    }
    ParseTree result = (ParseTree) startRule.invoke(parser, args);
    //		System.out.println("parse tree = "+result.toStringTree(parser));
    return result;
}