Examples with WritableToken org.antlr.v4.runtime.WritableToken used on opensource projects

Example 1 with WritableToken

use of org.antlr.v4.runtime.WritableToken in project titan.EclipsePlug-ins by eclipse.

the class ModuleLevelTokenStreamTracker method getBlock.

private boolean getBlock(final Token first) {
    // return true if EOF hit
    Token t;
    TokenWithIndexAndSubTokens result;
    t = getTokenSource().nextToken();
    if (t instanceof WritableToken) {
        ((WritableToken) t).setTokenIndex(tokens.size());
    }
    List<Token> tokenList = new ArrayList<Token>();
    int nofUnclosedParanthesis = 1;
    while (t != null && t.getType() != Token.EOF) {
        if (t.getType() == Asn1Lexer.BEGINCHAR) {
            nofUnclosedParanthesis++;
        } else if (t.getType() == Asn1Lexer.ENDCHAR) {
            nofUnclosedParanthesis--;
            if (nofUnclosedParanthesis == 0) {
                result = new TokenWithIndexAndSubTokens(Asn1Lexer.BLOCK, tokenList, sourceFile);
                result.setCharPositionInLine(first.getCharPositionInLine());
                result.setLine(first.getLine());
                result.setStartIndex(((TokenWithIndexAndSubTokens) first).getStopIndex());
                result.setStopIndex(((TokenWithIndexAndSubTokens) t).getStopIndex());
                result.setText(makeString(tokenList));
                tokens.add(result);
                return false;
            }
        }
        if (!discardMask.contains(Integer.valueOf(t.getType()))) {
            tokenList.add(new TokenWithIndexAndSubTokens(t));
        }
        t = getTokenSource().nextToken();
    }
    result = new TokenWithIndexAndSubTokens(Asn1Lexer.BLOCK, tokenList, sourceFile);
    result.setCharPositionInLine(first.getCharPositionInLine());
    result.setLine(first.getLine());
    result.setStartIndex(((TokenWithIndexAndSubTokens) first).getStopIndex());
    if (t != null) {
        result.setStopIndex(((TokenWithIndexAndSubTokens) t).getStopIndex());
    }
    tokens.add(result);
    return true;
}

Example 2 with WritableToken

use of org.antlr.v4.runtime.WritableToken in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testOffChannel.

@Test
public void testOffChannel() throws Exception {
    // simulate input " x =34  ;\n"
    TokenSource lexer = new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "x"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, "="), new CommonToken(1, "34"), new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(1, ";"), new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public TokenFactory getTokenFactory() {
            return CommonTokenFactory.DEFAULT;
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    // must skip first off channel token
    assertEquals("x", tokens.LT(1).getText());
    tokens.consume();
    assertEquals("=", tokens.LT(1).getText());
    assertEquals("x", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals("34", tokens.LT(1).getText());
    assertEquals("=", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(";", tokens.LT(1).getText());
    assertEquals("34", tokens.LT(-1).getText());
    tokens.consume();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(";", tokens.LT(-1).getText());
    assertEquals("34", tokens.LT(-2).getText());
    assertEquals("=", tokens.LT(-3).getText());
    assertEquals("x", tokens.LT(-4).getText());
}

Example 3 with WritableToken

use of org.antlr.v4.runtime.WritableToken in project antlr4 by tunnelvisionlabs.

the class TestCommonTokenStream method testFetchOffChannel.

@Test
public void testFetchOffChannel() throws Exception {
    // simulate input " x =34  ; \n"
    TokenSource lexer = // token indexes   01234 56789
    new TokenSource() {

        int i = 0;

        @SuppressWarnings("serial")
        WritableToken[] tokens = { // 0
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 1
        new CommonToken(1, "x"), // 2
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 3
        new CommonToken(1, "="), // 4
        new CommonToken(1, "34"), // 5
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 6
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 7
        new CommonToken(1, ";"), // 8
        new CommonToken(1, " ") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 9
        new CommonToken(1, "\n") {

            {
                channel = Lexer.HIDDEN;
            }
        }, // 10
        new CommonToken(Token.EOF, "") };

        @Override
        public Token nextToken() {
            return tokens[i++];
        }

        @Override
        public String getSourceName() {
            return "test";
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public TokenFactory getTokenFactory() {
            return CommonTokenFactory.DEFAULT;
        }

        @Override
        public void setTokenFactory(TokenFactory factory) {
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(null, tokens.getHiddenTokensToLeft(0));
    assertEquals(null, tokens.getHiddenTokensToRight(0));
    assertEquals("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(1).toString());
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(1).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(2));
    assertEquals(null, tokens.getHiddenTokensToRight(2));
    assertEquals("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(3).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(3));
    assertEquals(null, tokens.getHiddenTokensToLeft(4));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(4).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(5));
    assertEquals("[[@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(5).toString());
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(6).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(6));
    assertEquals("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(7).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(7).toString());
    assertEquals(null, tokens.getHiddenTokensToLeft(8));
    assertEquals("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToRight(8).toString());
    assertEquals("[[@8,0:0=' ',<1>,channel=1,0:-1]]", tokens.getHiddenTokensToLeft(9).toString());
    assertEquals(null, tokens.getHiddenTokensToRight(9));
}

Example 4 with WritableToken

use of org.antlr.v4.runtime.WritableToken in project titan.EclipsePlug-ins by eclipse.

the class ConfigTreeNodeUtilities method setText.

/**
 * Changes the text of a parse tree
 * @param aParseTree parse tree to modify
 * @param aText new text
 */
public static void setText(final ParseTree aParseTree, final String aText) {
    if (aParseTree == null) {
        ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.setText(): aParseTree == null");
        return;
    }
    if (aParseTree instanceof ParserRuleContext) {
        final ParserRuleContext rule = (ParserRuleContext) aParseTree;
        // just delete it and replace the children with an AddedParseTree
        if (rule.children != null) {
            rule.children.clear();
        } else {
            rule.children = new ArrayList<ParseTree>();
        }
        final ParseTree newNode = new AddedParseTree(aText);
        addChild(rule, newNode);
    } else if (aParseTree instanceof AddedParseTree) {
        final AddedParseTree node = (AddedParseTree) aParseTree;
        node.setText(aText);
    } else if (aParseTree instanceof TerminalNodeImpl) {
        final TerminalNodeImpl node = (TerminalNodeImpl) aParseTree;
        final Token t = node.symbol;
        if (t instanceof WritableToken) {
            final WritableToken ct = (WritableToken) t;
            ct.setText(aText);
        } else {
            ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.setText(): unhandled token class type");
        }
    } else {
        ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.setText(): unhandled ParseTree class type");
    }
}

Example 5 with WritableToken

use of org.antlr.v4.runtime.WritableToken in project titan.EclipsePlug-ins by eclipse.

the class ModuleLevelTokenStreamTracker method fetch.

@Override
public int fetch(int n) {
    if (fetchedEOF) {
        return 0;
    }
    Token t;
    Token first;
    int i = 0;
    do {
        t = getTokenSource().nextToken();
        if (t instanceof WritableToken) {
            ((WritableToken) t).setTokenIndex(tokens.size());
        }
        first = t;
        if (t.getType() == Token.EOF) {
            fetchedEOF = true;
            tokens.add(new TokenWithIndexAndSubTokens(t));
            return ++i;
        } else if (discardMask.contains(Integer.valueOf(t.getType()))) {
            // discard this Token
            assert (true);
        } else if (t.getType() == Asn1Lexer.BEGINCHAR) {
            fetchedEOF = getBlock(first);
            if (fetchedEOF) {
                return ++i;
            }
            ++i;
            --n;
        } else {
            tokens.add(t);
            ++i;
            --n;
        }
    } while (0 < n);
    return i;
}