Search in sources :

Example 6 with LexerGrammar

use of org.antlr.v4.tool.LexerGrammar in project antlr4 by antlr.

the class BaseGoTest method createATN.

protected ATN createATN(Grammar g, boolean useSerializer) {
    if (g.atn == null) {
        semanticProcess(g);
        assertEquals(0, g.tool.getNumErrors());
        ParserATNFactory f;
        if (g.isLexer()) {
            f = new LexerATNFactory((LexerGrammar) g);
        } else {
            f = new ParserATNFactory(g);
        }
        g.atn = f.createATN();
        assertEquals(0, g.tool.getNumErrors());
    }
    ATN atn = g.atn;
    if (useSerializer) {
        char[] serialized = ATNSerializer.getSerializedAsChars(atn);
        return new ATNDeserializer().deserialize(serialized);
    }
    return atn;
}
Also used : ATNDeserializer(org.antlr.v4.runtime.atn.ATNDeserializer) ParserATNFactory(org.antlr.v4.automata.ParserATNFactory) ATN(org.antlr.v4.runtime.atn.ATN) LexerGrammar(org.antlr.v4.tool.LexerGrammar) LexerATNFactory(org.antlr.v4.automata.LexerATNFactory)

Example 7 with LexerGrammar

use of org.antlr.v4.tool.LexerGrammar in project antlr4 by antlr.

the class BaseBrowserTest method getTokenTypes.

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}
Also used : LexerATNSimulator(org.antlr.v4.runtime.atn.LexerATNSimulator) ArrayList(java.util.ArrayList) STGroupString(org.stringtemplate.v4.STGroupString) BaseRuntimeTest.antlrOnString(org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString) DFA(org.antlr.v4.runtime.dfa.DFA)

Example 8 with LexerGrammar

use of org.antlr.v4.tool.LexerGrammar in project antlr4 by antlr.

the class BaseNodeTest method testActions.

public void testActions(String templates, String actionName, String action, String expected) throws org.antlr.runtime.RecognitionException {
    int lp = templates.indexOf('(');
    String name = templates.substring(0, lp);
    STGroup group = new STGroupString(templates);
    ST st = group.getInstanceOf(name);
    st.add(actionName, action);
    String grammar = st.render();
    ErrorQueue equeue = new ErrorQueue();
    Grammar g = new Grammar(grammar, equeue);
    if (g.ast != null && !g.ast.hasErrors) {
        SemanticPipeline sem = new SemanticPipeline(g);
        sem.process();
        ATNFactory factory = new ParserATNFactory(g);
        if (g.isLexer())
            factory = new LexerATNFactory((LexerGrammar) g);
        g.atn = factory.createATN();
        CodeGenerator gen = new CodeGenerator(g);
        ST outputFileST = gen.generateParser();
        String output = outputFileST.render();
        // System.out.println(output);
        String b = "#" + actionName + "#";
        int start = output.indexOf(b);
        String e = "#end-" + actionName + "#";
        int end = output.indexOf(e);
        String snippet = output.substring(start + b.length(), end);
        assertEquals(expected, snippet);
    }
    if (equeue.size() > 0) {
        System.err.println(equeue.toString());
    }
}
Also used : ST(org.stringtemplate.v4.ST) SemanticPipeline(org.antlr.v4.semantics.SemanticPipeline) ParserATNFactory(org.antlr.v4.automata.ParserATNFactory) STGroup(org.stringtemplate.v4.STGroup) ParserATNFactory(org.antlr.v4.automata.ParserATNFactory) ATNFactory(org.antlr.v4.automata.ATNFactory) LexerATNFactory(org.antlr.v4.automata.LexerATNFactory) ErrorQueue(org.antlr.v4.test.runtime.ErrorQueue) STGroupString(org.stringtemplate.v4.STGroupString) BaseRuntimeTest.antlrOnString(org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString) Grammar(org.antlr.v4.tool.Grammar) LexerGrammar(org.antlr.v4.tool.LexerGrammar) CodeGenerator(org.antlr.v4.codegen.CodeGenerator) LexerATNFactory(org.antlr.v4.automata.LexerATNFactory) STGroupString(org.stringtemplate.v4.STGroupString)

Example 9 with LexerGrammar

use of org.antlr.v4.tool.LexerGrammar in project antlr4 by antlr.

the class BaseNodeTest method createATN.

protected ATN createATN(Grammar g, boolean useSerializer) {
    if (g.atn == null) {
        semanticProcess(g);
        assertEquals(0, g.tool.getNumErrors());
        ParserATNFactory f;
        if (g.isLexer()) {
            f = new LexerATNFactory((LexerGrammar) g);
        } else {
            f = new ParserATNFactory(g);
        }
        g.atn = f.createATN();
        assertEquals(0, g.tool.getNumErrors());
    }
    ATN atn = g.atn;
    if (useSerializer) {
        char[] serialized = ATNSerializer.getSerializedAsChars(atn);
        return new ATNDeserializer().deserialize(serialized);
    }
    return atn;
}
Also used : ATNDeserializer(org.antlr.v4.runtime.atn.ATNDeserializer) ParserATNFactory(org.antlr.v4.automata.ParserATNFactory) ATN(org.antlr.v4.runtime.atn.ATN) LexerGrammar(org.antlr.v4.tool.LexerGrammar) LexerATNFactory(org.antlr.v4.automata.LexerATNFactory)

Example 10 with LexerGrammar

use of org.antlr.v4.tool.LexerGrammar in project antlr4 by antlr.

the class BaseNodeTest method getTokenTypes.

public List<String> getTokenTypes(LexerGrammar lg, ATN atn, CharStream input) {
    LexerATNSimulator interp = new LexerATNSimulator(atn, new DFA[] { new DFA(atn.modeToStartState.get(Lexer.DEFAULT_MODE)) }, null);
    List<String> tokenTypes = new ArrayList<String>();
    int ttype;
    boolean hitEOF = false;
    do {
        if (hitEOF) {
            tokenTypes.add("EOF");
            break;
        }
        int t = input.LA(1);
        ttype = interp.match(input, Lexer.DEFAULT_MODE);
        if (ttype == Token.EOF) {
            tokenTypes.add("EOF");
        } else {
            tokenTypes.add(lg.typeToTokenList.get(ttype));
        }
        if (t == IntStream.EOF) {
            hitEOF = true;
        }
    } while (ttype != Token.EOF);
    return tokenTypes;
}
Also used : LexerATNSimulator(org.antlr.v4.runtime.atn.LexerATNSimulator) ArrayList(java.util.ArrayList) STGroupString(org.stringtemplate.v4.STGroupString) BaseRuntimeTest.antlrOnString(org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString) DFA(org.antlr.v4.runtime.dfa.DFA)

Aggregations

LexerGrammar (org.antlr.v4.tool.LexerGrammar)444 Test (org.junit.Test)410 Grammar (org.antlr.v4.tool.Grammar)140 LexerInterpreter (org.antlr.v4.runtime.LexerInterpreter)120 CommonTokenStream (org.antlr.v4.runtime.CommonTokenStream)100 TokenStreamRewriter (org.antlr.v4.runtime.TokenStreamRewriter)86 ATN (org.antlr.v4.runtime.atn.ATN)72 ANTLRInputStream (org.antlr.v4.runtime.ANTLRInputStream)58 BaseJavaTest (org.antlr.v4.test.runtime.java.BaseJavaTest)43 ParserATNFactory (org.antlr.v4.automata.ParserATNFactory)26 ParseTree (org.antlr.v4.runtime.tree.ParseTree)24 LexerATNFactory (org.antlr.v4.automata.LexerATNFactory)22 CharStream (org.antlr.v4.runtime.CharStream)20 TokenStream (org.antlr.v4.runtime.TokenStream)17 LexerATNSimulator (org.antlr.v4.runtime.atn.LexerATNSimulator)14 STGroupString (org.stringtemplate.v4.STGroupString)14 ArrayList (java.util.ArrayList)13 ATNFactory (org.antlr.v4.automata.ATNFactory)12 DFA (org.antlr.v4.runtime.dfa.DFA)12 SemanticPipeline (org.antlr.v4.semantics.SemanticPipeline)12