Examples with CommonToken org.antlr.v4.runtime.CommonToken used on opensource projects

Example 41 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project checkstyle by checkstyle.

the class ParseTreeBuilder method treeFewWhiteSpacesAsSeparator.

public static ParseTree treeFewWhiteSpacesAsSeparator() {
    JavadocContext javadocContext0 = new JavadocContext(null, 0);
    CommonToken leadingAsterisk0 = new CommonToken(JavadocTokenTypes.LEADING_ASTERISK, "       *");
    javadocContext0.addChild(leadingAsterisk0);
    CommonToken ws0 = new CommonToken(JavadocTokenTypes.WS, " ");
    javadocContext0.addChild(ws0);
    JavadocTagContext javadocTagContext0 = new JavadocTagContext(javadocContext0, 0);
    CommonToken paramLiteral0 = new CommonToken(JavadocTokenTypes.PARAM_LITERAL, "@param");
    javadocTagContext0.addChild(paramLiteral0);
    CommonToken ws1 = new CommonToken(JavadocTokenTypes.WS, "   ");
    javadocTagContext0.addChild(ws1);
    CommonToken parameterName0 = new CommonToken(JavadocTokenTypes.PARAMETER_NAME, "initialCapacity");
    javadocTagContext0.addChild(parameterName0);
    CommonToken ws2 = new CommonToken(JavadocTokenTypes.WS, "     ");
    javadocTagContext0.addChild(ws2);
    DescriptionContext descriptionContext0 = new DescriptionContext(javadocTagContext0, 0);
    TextContext textContext0 = new TextContext(descriptionContext0, 0);
    CommonToken char0 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext0.addChild(char0);
    CommonToken char1 = new CommonToken(JavadocTokenTypes.CHAR, "h");
    textContext0.addChild(char1);
    CommonToken char2 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext0.addChild(char2);
    CommonToken ws3 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws3);
    CommonToken char3 = new CommonToken(JavadocTokenTypes.CHAR, "i");
    textContext0.addChild(char3);
    CommonToken char4 = new CommonToken(JavadocTokenTypes.CHAR, "n");
    textContext0.addChild(char4);
    CommonToken char5 = new CommonToken(JavadocTokenTypes.CHAR, "i");
    textContext0.addChild(char5);
    CommonToken char6 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext0.addChild(char6);
    CommonToken char7 = new CommonToken(JavadocTokenTypes.CHAR, "i");
    textContext0.addChild(char7);
    CommonToken char8 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext0.addChild(char8);
    CommonToken char9 = new CommonToken(JavadocTokenTypes.CHAR, "l");
    textContext0.addChild(char9);
    CommonToken ws4 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws4);
    CommonToken char10 = new CommonToken(JavadocTokenTypes.CHAR, "c");
    textContext0.addChild(char10);
    CommonToken char11 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext0.addChild(char11);
    CommonToken char12 = new CommonToken(JavadocTokenTypes.CHAR, "p");
    textContext0.addChild(char12);
    CommonToken char13 = new CommonToken(JavadocTokenTypes.CHAR, "a");
    textContext0.addChild(char13);
    CommonToken char14 = new CommonToken(JavadocTokenTypes.CHAR, "c");
    textContext0.addChild(char14);
    CommonToken char15 = new CommonToken(JavadocTokenTypes.CHAR, "i");
    textContext0.addChild(char15);
    CommonToken char16 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext0.addChild(char16);
    CommonToken char17 = new CommonToken(JavadocTokenTypes.CHAR, "y");
    textContext0.addChild(char17);
    CommonToken ws5 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws5);
    CommonToken char18 = new CommonToken(JavadocTokenTypes.CHAR, "o");
    textContext0.addChild(char18);
    CommonToken char19 = new CommonToken(JavadocTokenTypes.CHAR, "f");
    textContext0.addChild(char19);
    CommonToken ws6 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws6);
    CommonToken char20 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext0.addChild(char20);
    CommonToken char21 = new CommonToken(JavadocTokenTypes.CHAR, "h");
    textContext0.addChild(char21);
    CommonToken char22 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext0.addChild(char22);
    CommonToken ws7 = new CommonToken(JavadocTokenTypes.WS, " ");
    textContext0.addChild(ws7);
    CommonToken char23 = new CommonToken(JavadocTokenTypes.CHAR, "B");
    textContext0.addChild(char23);
    CommonToken char24 = new CommonToken(JavadocTokenTypes.CHAR, "y");
    textContext0.addChild(char24);
    CommonToken char25 = new CommonToken(JavadocTokenTypes.CHAR, "t");
    textContext0.addChild(char25);
    CommonToken char26 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext0.addChild(char26);
    CommonToken char27 = new CommonToken(JavadocTokenTypes.CHAR, "B");
    textContext0.addChild(char27);
    CommonToken char28 = new CommonToken(JavadocTokenTypes.CHAR, "u");
    textContext0.addChild(char28);
    CommonToken char29 = new CommonToken(JavadocTokenTypes.CHAR, "f");
    textContext0.addChild(char29);
    CommonToken char30 = new CommonToken(JavadocTokenTypes.CHAR, "f");
    textContext0.addChild(char30);
    CommonToken char31 = new CommonToken(JavadocTokenTypes.CHAR, "e");
    textContext0.addChild(char31);
    CommonToken char32 = new CommonToken(JavadocTokenTypes.CHAR, "r");
    textContext0.addChild(char32);
    CommonToken char33 = new CommonToken(JavadocTokenTypes.CHAR, ".");
    textContext0.addChild(char33);
    descriptionContext0.addChild(textContext0);
    javadocTagContext0.addChild(descriptionContext0);
    javadocContext0.addChild(javadocTagContext0);
    CommonToken eof0 = new CommonToken(JavadocTokenTypes.EOF, "<EOF>");
    javadocContext0.addChild(eof0);
    return javadocContext0;
}

Example 42 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class Trees method stripChildrenOutOfRange.

/** Replace any subtree siblings of root that are completely to left
	 *  or right of lookahead range with a CommonToken(Token.INVALID_TYPE,"...")
	 *  node. The source interval for t is not altered to suit smaller range!
	 *
	 *  WARNING: destructive to t.
	 *
	 *  @since 4.5.1
	 */
public static void stripChildrenOutOfRange(ParserRuleContext t, ParserRuleContext root, int startIndex, int stopIndex) {
    if (t == null)
        return;
    for (int i = 0; i < t.getChildCount(); i++) {
        ParseTree child = t.getChild(i);
        Interval range = child.getSourceInterval();
        if (child instanceof ParserRuleContext && (range.b < startIndex || range.a > stopIndex)) {
            if (isAncestorOf(child, root)) {
                // replace only if subtree doesn't have displayed root
                CommonToken abbrev = new CommonToken(Token.INVALID_TYPE, "...");
                t.children.set(i, new TerminalNodeImpl(abbrev));
            }
        }
    }
}

Example 43 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class LexerATNFactory method action.

@Override
public Handle action(String action) {
    if (action.trim().isEmpty()) {
        ATNState left = newState(null);
        ATNState right = newState(null);
        epsilon(left, right);
        return new Handle(left, right);
    }
    // define action AST for this rule as if we had found in grammar
    ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action));
    currentRule.defineActionInAlt(currentOuterAlt, ast);
    return action(ast);
}

Example 44 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 45 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            return null;
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}