Examples with CommonToken org.antlr.v4.runtime.CommonToken used on opensource projects

Example 21 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class TestCommonTokenStream method testSingleEOF.

@Test
public void testSingleEOF() throws Exception {
    TokenSource lexer = new TokenSource() {

        @Override
        public Token nextToken() {
            return new CommonToken(Token.EOF);
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public String getSourceName() {
            return IntStream.UNKNOWN_SOURCE_NAME;
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
}

Example 22 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class TestCommonTokenStream method testCannotConsumeEOF.

@Test(expected = IllegalStateException.class)
public void testCannotConsumeEOF() throws Exception {
    TokenSource lexer = new TokenSource() {

        @Override
        public Token nextToken() {
            return new CommonToken(Token.EOF);
        }

        @Override
        public int getLine() {
            return 0;
        }

        @Override
        public int getCharPositionInLine() {
            return 0;
        }

        @Override
        public CharStream getInputStream() {
            return null;
        }

        @Override
        public String getSourceName() {
            return IntStream.UNKNOWN_SOURCE_NAME;
        }

        @Override
        public TokenFactory<?> getTokenFactory() {
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public void setTokenFactory(TokenFactory<?> factory) {
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.fill();
    assertEquals(Token.EOF, tokens.LA(1));
    assertEquals(0, tokens.index());
    assertEquals(1, tokens.size());
    tokens.consume();
}

Example 23 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class GrammarASTAdaptor method create.

@Override
public /** Make sure even imaginary nodes know the input stream */
Object create(int tokenType, String text) {
    GrammarAST t;
    if (tokenType == ANTLRParser.RULE) {
        // needed by TreeWizard to make RULE tree
        t = new RuleAST(new CommonToken(tokenType, text));
    } else if (tokenType == ANTLRParser.STRING_LITERAL) {
        // implicit lexer construction done with wizard; needs this node type
        // whereas grammar ANTLRParser.g can use token option to spec node type
        t = new TerminalAST(new CommonToken(tokenType, text));
    } else {
        t = (GrammarAST) super.create(tokenType, text);
    }
    t.token.setInputStream(input);
    return t;
}

Example 24 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project antlr4 by antlr.

the class LeftRecursiveRuleAnalyzer method addPrecedenceArgToRules.

public AltAST addPrecedenceArgToRules(AltAST t, int prec) {
    if (t == null)
        return null;
    // get all top-level rule refs from ALT
    List<GrammarAST> outerAltRuleRefs = t.getNodesWithTypePreorderDFS(IntervalSet.of(RULE_REF));
    for (GrammarAST x : outerAltRuleRefs) {
        RuleRefAST rref = (RuleRefAST) x;
        boolean recursive = rref.getText().equals(ruleName);
        boolean rightmost = rref == outerAltRuleRefs.get(outerAltRuleRefs.size() - 1);
        if (recursive && rightmost) {
            GrammarAST dummyValueNode = new GrammarAST(new CommonToken(ANTLRParser.INT, "" + prec));
            rref.setOption(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME, dummyValueNode);
        }
    }
    return t;
}

Example 25 with CommonToken

use of org.antlr.v4.runtime.CommonToken in project sqldelight by square.

the class TokenStreamSubset method setIndexOfLastToken.

public void setIndexOfLastToken(int indexOfLastToken) {
    System.out.println("setIndexOfLastToken(" + indexOfLastToken + ")");
    if (indexOfLastToken < 0) {
        System.out.println("replacing " + saveToken.getTokenIndex() + " with " + saveToken);
        tokens.set(saveToken.getTokenIndex(), saveToken);
        //			this.indexOfLastToken = indexOfLastToken;
        return;
    }
    // we want to keep token at indexOfLastToken
    int i = indexOfLastToken + 1;
    sync(i);
    saveToken = tokens.get(i);
    System.out.println("saving " + saveToken);
    CommonToken stopToken = new CommonToken(saveToken);
    stopToken.setType(STOP_TOKEN_TYPE);
    System.out.println("setting " + i + " to " + stopToken);
    tokens.set(i, stopToken);
//		this.indexOfLastToken = indexOfLastToken;
}