Examples with CommonToken org.antlr.v4.runtime.CommonToken used on opensource projects

Example 16 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project checkstyle by checkstyle.

the class ParseTreeBuilder method treeJavadocTagDescriptionWithInlineTags.

public static ParseTree treeJavadocTagDescriptionWithInlineTags() {
    JavadocContext javadocContext0 = new JavadocContext(null, 0);
    JavadocTagContext javadocTagContext0 = new JavadocTagContext(javadocContext0, 0);
    CommonToken deprecatedLiteral0 = new CommonToken(JavadocTokenTypes.DEPRECATED_LITERAL, "@deprecated");
    javadocTagContext0.addChild(deprecatedLiteral0);
    CommonToken ws0 = new CommonToken(JavadocTokenTypes.WS, " ");
    javadocTagContext0.addChild(ws0);
    DescriptionContext descriptionContext0 = new DescriptionContext(javadocTagContext0, 0);
    TextContext textContext0 = new TextContext(descriptionContext0, 0);
    CommonToken char0 = new CommonToken(JavadocTokenTypes.CHAR, "u");
    textContext0.addChild(char0);
    CommonToken char1 = new CommonToken(JavadocTokenTypes.CHAR, "s");
    textContext0.addChild(char1);
    CommonToken char2 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext0.addChild(char2);
    CommonToken ws1 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws1);
    descriptionContext0.addChild(textContext0);
    JavadocInlineTagContext javadocInlineTagContext0 = new JavadocInlineTagContext(descriptionContext0, 0);
    CommonToken javadocInlineTagStart0 = new CommonToken(JavadocTokenTypes.JAVADOC_INLINE_TAG_START, "{");
    javadocInlineTagContext0.addChild(javadocInlineTagStart0);
    CommonToken linkLiteral0 = new CommonToken(JavadocTokenTypes.LINK_LITERAL, "@link");
    javadocInlineTagContext0.addChild(linkLiteral0);
    CommonToken ws2 = new CommonToken(JavadocTokenTypes.WS, " ");
    javadocInlineTagContext0.addChild(ws2);
    ReferenceContext referenceContext0 = new ReferenceContext(javadocInlineTagContext0, 0);
    CommonToken package0 = new CommonToken(JavadocTokenTypes.PACKAGE, "java.lang");
    referenceContext0.addChild(package0);
    CommonToken dot0 = new CommonToken(JavadocTokenTypes.DOT, ".");
    referenceContext0.addChild(dot0);
    CommonToken class0 = new CommonToken(JavadocTokenTypes.CLASS, "Integer");
    referenceContext0.addChild(class0);
    javadocInlineTagContext0.addChild(referenceContext0);
    DescriptionContext descriptionContext1 = new DescriptionContext(javadocInlineTagContext0, 0);
    TextContext textContext1 = new TextContext(descriptionContext1, 0);
    CommonToken ws3 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext1.addChild(ws3);
    CommonToken char3 = new CommonToken(JavadocTokenTypes.CHAR, "I");
    textContext1.addChild(char3);
    CommonToken char4 = new CommonToken(JavadocTokenTypes.CHAR, "n");
    textContext1.addChild(char4);
    CommonToken char5 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext1.addChild(char5);
    CommonToken char6 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext1.addChild(char6);
    CommonToken char7 = new CommonToken(JavadocTokenTypes.CHAR, "g");
    textContext1.addChild(char7);
    CommonToken char8 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext1.addChild(char8);
    CommonToken char9 = new CommonToken(JavadocTokenTypes.CHAR, "r");
    textContext1.addChild(char9);
    descriptionContext1.addChild(textContext1);
    JavadocInlineTagContext javadocInlineTagContext1 = new JavadocInlineTagContext(descriptionContext1, 0);
    CommonToken javadocInlineTagStart1 = new CommonToken(JavadocTokenTypes.JAVADOC_INLINE_TAG_START, "{");
    javadocInlineTagContext1.addChild(javadocInlineTagStart1);
    CommonToken codeLiteral0 = new CommonToken(JavadocTokenTypes.CODE_LITERAL, "@code");
    javadocInlineTagContext1.addChild(codeLiteral0);
    CommonToken ws4 = new CommonToken(JavadocTokenTypes.WS, " ");
    javadocInlineTagContext1.addChild(ws4);
    TextContext textContext2 = new TextContext(javadocInlineTagContext1, 0);
    CommonToken char10 = new CommonToken(JavadocTokenTypes.CHAR, "<");
    textContext2.addChild(char10);
    CommonToken char11 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext2.addChild(char11);
    CommonToken char12 = new CommonToken(JavadocTokenTypes.CHAR, ">");
    textContext2.addChild(char12);
    CommonToken char13 = new CommonToken(JavadocTokenTypes.CHAR, "W");
    textContext2.addChild(char13);
    CommonToken char14 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext2.addChild(char14);
    CommonToken char15 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext2.addChild(char15);
    CommonToken char16 = new CommonToken(JavadocTokenTypes.CHAR, "<");
    textContext2.addChild(char16);
    CommonToken char17 = new CommonToken(JavadocTokenTypes.CHAR, "/");
    textContext2.addChild(char17);
    CommonToken char18 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext2.addChild(char18);
    CommonToken char19 = new CommonToken(JavadocTokenTypes.CHAR, ">");
    textContext2.addChild(char19);
    javadocInlineTagContext1.addChild(textContext2);
    CommonToken javadocInlineTagEnd0 = new CommonToken(JavadocTokenTypes.JAVADOC_INLINE_TAG_END, "}");
    javadocInlineTagContext1.addChild(javadocInlineTagEnd0);
    descriptionContext1.addChild(javadocInlineTagContext1);
    javadocInlineTagContext0.addChild(descriptionContext1);
    CommonToken javadocInlineTagEnd1 = new CommonToken(JavadocTokenTypes.JAVADOC_INLINE_TAG_END, "}");
    javadocInlineTagContext0.addChild(javadocInlineTagEnd1);
    descriptionContext0.addChild(javadocInlineTagContext0);
    TextContext textContext3 = new TextContext(descriptionContext0, 0);
    CommonToken ws5 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext3.addChild(ws5);
    CommonToken char20 = new CommonToken(JavadocTokenTypes.CHAR, "i");
    textContext3.addChild(char20);
    CommonToken char21 = new CommonToken(JavadocTokenTypes.CHAR, "n");
    textContext3.addChild(char21);
    CommonToken char22 = new CommonToken(JavadocTokenTypes.CHAR, "s");
    textContext3.addChild(char22);
    CommonToken char23 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext3.addChild(char23);
    CommonToken char24 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext3.addChild(char24);
    CommonToken char25 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext3.addChild(char25);
    CommonToken char26 = new CommonToken(JavadocTokenTypes.CHAR, "d");
    textContext3.addChild(char26);
    descriptionContext0.addChild(textContext3);
    javadocTagContext0.addChild(descriptionContext0);
    javadocContext0.addChild(javadocTagContext0);
    CommonToken eof0 = new CommonToken(JavadocTokenTypes.EOF, "<EOF>");
    javadocContext0.addChild(eof0);
    return javadocContext0;
}

Example 17 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project pinot by linkedin.

the class Pql2Compiler method compileToBrokerRequest.

@Override
public BrokerRequest compileToBrokerRequest(String expression) throws Pql2CompilationException {
    try {
        //
        CharStream charStream = new ANTLRInputStream(expression);
        PQL2Lexer lexer = new PQL2Lexer(charStream);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
        PQL2Parser parser = new PQL2Parser(tokenStream);
        parser.setErrorHandler(new BailErrorStrategy());
        // Parse
        ParseTree parseTree = parser.root();
        ParseTreeWalker walker = new ParseTreeWalker();
        Pql2AstListener listener = new Pql2AstListener(expression);
        walker.walk(listener, parseTree);
        AstNode rootNode = listener.getRootNode();
        BrokerRequest brokerRequest = new BrokerRequest();
        rootNode.updateBrokerRequest(brokerRequest);
        return brokerRequest;
    } catch (Pql2CompilationException e) {
        throw e;
    } catch (Exception e) {
        throw new Pql2CompilationException(e.getMessage());
    }
}

Example 18 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class XPathLexer method nextToken.

@Override
public Token nextToken() {
    _tokenStartCharIndex = _input.index();
    CommonToken t = null;
    while (t == null) {
        switch(_input.LA(1)) {
            case '/':
                consume();
                if (_input.LA(1) == '/') {
                    consume();
                    t = new CommonToken(ANYWHERE, "//");
                } else {
                    t = new CommonToken(ROOT, "/");
                }
                break;
            case '*':
                consume();
                t = new CommonToken(WILDCARD, "*");
                break;
            case '!':
                consume();
                t = new CommonToken(BANG, "!");
                break;
            case '\'':
                String s = matchString();
                t = new CommonToken(STRING, s);
                break;
            case CharStream.EOF:
                return new CommonToken(EOF, "<EOF>");
            default:
                if (isNameStartChar(_input.LA(1))) {
                    String id = matchID();
                    if (Character.isUpperCase(id.charAt(0)))
                        t = new CommonToken(TOKEN_REF, id);
                    else
                        t = new CommonToken(RULE_REF, id);
                } else {
                    throw new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null);
                }
                break;
        }
    }
    t.setStartIndex(_tokenStartCharIndex);
    t.setCharPositionInLine(_tokenStartCharIndex);
    t.setLine(line);
    return t;
}

Example 19 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class DefaultErrorStrategy method getMissingSymbol.

/** Conjure up a missing token during error recovery.
	 *
	 *  The recognizer attempts to recover from single missing
	 *  symbols. But, actions might refer to that missing symbol.
	 *  For example, x=ID {f($x);}. The action clearly assumes
	 *  that there has been an identifier matched previously and that
	 *  $x points at that token. If that token is missing, but
	 *  the next token in the stream is what we want we assume that
	 *  this token is missing and we keep going. Because we
	 *  have to return some token to replace the missing token,
	 *  we have to conjure one up. This method gives the user control
	 *  over the tokens returned for missing tokens. Mostly,
	 *  you will want to create something special for identifier
	 *  tokens. For literals such as '{' and ',', the default
	 *  action in the parser or tree parser works. It simply creates
	 *  a CommonToken of the appropriate type. The text will be the token.
	 *  If you change what tokens must be created by the lexer,
	 *  override this method to create the appropriate tokens.
	 */
protected Token getMissingSymbol(Parser recognizer) {
    Token currentSymbol = recognizer.getCurrentToken();
    IntervalSet expecting = getExpectedTokens(recognizer);
    int expectedTokenType = Token.INVALID_TYPE;
    if (!expecting.isNil()) {
        // get any element
        expectedTokenType = expecting.getMinElement();
    }
    String tokenText;
    if (expectedTokenType == Token.EOF)
        tokenText = "<missing EOF>";
    else
        tokenText = "<missing " + recognizer.getVocabulary().getDisplayName(expectedTokenType) + ">";
    Token current = currentSymbol;
    Token lookback = recognizer.getInputStream().LT(-1);
    if (current.getType() == Token.EOF && lookback != null) {
        current = lookback;
    }
    return recognizer.getTokenFactory().create(new Pair<TokenSource, CharStream>(current.getTokenSource(), current.getTokenSource().getInputStream()), expectedTokenType, tokenText, Token.DEFAULT_CHANNEL, -1, -1, current.getLine(), current.getCharPositionInLine());
}

Example 20 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class GrammarTransformPipeline method extractImplicitLexer.

/** Build lexer grammar from combined grammar that looks like:
	 *
	 *  (COMBINED_GRAMMAR A
	 *      (tokens { X (= Y 'y'))
	 *      (OPTIONS (= x 'y'))
	 *      (@ members {foo})
	 *      (@ lexer header {package jj;})
	 *      (RULES (RULE .+)))
	 *
	 *  Move rules and actions to new tree, don't dup. Split AST apart.
	 *  We'll have this Grammar share token symbols later; don't generate
	 *  tokenVocab or tokens{} section.  Copy over named actions.
	 *
	 *  Side-effects: it removes children from GRAMMAR & RULES nodes
	 *                in combined AST.  Anything cut out is dup'd before
	 *                adding to lexer to avoid "who's ur daddy" issues
	 */
public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
    GrammarRootAST combinedAST = combinedGrammar.ast;
    //tool.log("grammar", "before="+combinedAST.toStringTree());
    GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.token.getInputStream());
    GrammarAST[] elements = combinedAST.getChildren().toArray(new GrammarAST[0]);
    // MAKE A GRAMMAR ROOT and ID
    String lexerName = combinedAST.getChild(0).getText() + "Lexer";
    GrammarRootAST lexerAST = new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
    lexerAST.grammarType = ANTLRParser.LEXER;
    lexerAST.token.setInputStream(combinedAST.token.getInputStream());
    lexerAST.addChild((GrammarAST) adaptor.create(ANTLRParser.ID, lexerName));
    // COPY OPTIONS
    GrammarAST optionsRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.OPTIONS);
    if (optionsRoot != null && optionsRoot.getChildCount() != 0) {
        GrammarAST lexerOptionsRoot = (GrammarAST) adaptor.dupNode(optionsRoot);
        lexerAST.addChild(lexerOptionsRoot);
        GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]);
        for (GrammarAST o : options) {
            String optionName = o.getChild(0).getText();
            if (Grammar.lexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName)) {
                GrammarAST optionTree = (GrammarAST) adaptor.dupTree(o);
                lexerOptionsRoot.addChild(optionTree);
                lexerAST.setOption(optionName, (GrammarAST) optionTree.getChild(1));
            }
        }
    }
    // COPY all named actions, but only move those with lexer:: scope
    List<GrammarAST> actionsWeMoved = new ArrayList<GrammarAST>();
    for (GrammarAST e : elements) {
        if (e.getType() == ANTLRParser.AT) {
            lexerAST.addChild((Tree) adaptor.dupTree(e));
            if (e.getChild(0).getText().equals("lexer")) {
                actionsWeMoved.add(e);
            }
        }
    }
    for (GrammarAST r : actionsWeMoved) {
        combinedAST.deleteChild(r);
    }
    GrammarAST combinedRulesRoot = (GrammarAST) combinedAST.getFirstChildWithType(ANTLRParser.RULES);
    if (combinedRulesRoot == null)
        return lexerAST;
    // MOVE lexer rules
    GrammarAST lexerRulesRoot = (GrammarAST) adaptor.create(ANTLRParser.RULES, "RULES");
    lexerAST.addChild(lexerRulesRoot);
    List<GrammarAST> rulesWeMoved = new ArrayList<GrammarAST>();
    GrammarASTWithOptions[] rules;
    if (combinedRulesRoot.getChildCount() > 0) {
        rules = combinedRulesRoot.getChildren().toArray(new GrammarASTWithOptions[0]);
    } else {
        rules = new GrammarASTWithOptions[0];
    }
    for (GrammarASTWithOptions r : rules) {
        String ruleName = r.getChild(0).getText();
        if (Grammar.isTokenName(ruleName)) {
            lexerRulesRoot.addChild((Tree) adaptor.dupTree(r));
            rulesWeMoved.add(r);
        }
    }
    for (GrammarAST r : rulesWeMoved) {
        combinedRulesRoot.deleteChild(r);
    }
    // Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
    List<Pair<GrammarAST, GrammarAST>> litAliases = Grammar.getStringLiteralAliasesFromLexerRules(lexerAST);
    Set<String> stringLiterals = combinedGrammar.getStringLiterals();
    // add strings from combined grammar (and imported grammars) into lexer
    // put them first as they are keywords; must resolve ambigs to these rules
    //		tool.log("grammar", "strings from parser: "+stringLiterals);
    int insertIndex = 0;
    nextLit: for (String lit : stringLiterals) {
        // if lexer already has a rule for literal, continue
        if (litAliases != null) {
            for (Pair<GrammarAST, GrammarAST> pair : litAliases) {
                GrammarAST litAST = pair.b;
                if (lit.equals(litAST.getText()))
                    continue nextLit;
            }
        }
        // create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
        String rname = combinedGrammar.getStringLiteralLexerRuleName(lit);
        // can't use wizard; need special node types
        GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
        BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
        AltAST alt = new AltAST(ANTLRParser.ALT);
        TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
        alt.addChild(slit);
        blk.addChild(alt);
        CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
        litRule.addChild(new TerminalAST(idToken));
        litRule.addChild(blk);
        lexerRulesRoot.insertChild(insertIndex, litRule);
        //			lexerRulesRoot.getChildren().add(0, litRule);
        // reset indexes and set litRule parent
        lexerRulesRoot.freshenParentAndChildIndexes();
        // next literal will be added after the one just added
        insertIndex++;
    }
    // TODO: take out after stable if slow
    lexerAST.sanityCheckParentAndChildIndexes();
    combinedAST.sanityCheckParentAndChildIndexes();
    //		tool.log("grammar", combinedAST.toTokenString());
    combinedGrammar.tool.log("grammar", "after extract implicit lexer =" + combinedAST.toStringTree());
    combinedGrammar.tool.log("grammar", "lexer =" + lexerAST.toStringTree());
    if (lexerRulesRoot.getChildCount() == 0)
        return null;
    return lexerAST;
}