Examples with Token groovyjarjarantlr.Token used on opensource projects

Example 1 with Token

use of groovyjarjarantlr.Token in project pmd by pmd.

the class GroovyTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    GroovyLexer lexer = new GroovyLexer(IOUtils.toInputStream(buffer.toString()));
    TokenStream tokenStream = lexer.plumb();
    try {
        Token token = tokenStream.nextToken();
        while (token.getType() != Token.EOF_TYPE) {
            TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine());
            tokenEntries.add(tokenEntry);
            token = tokenStream.nextToken();
        }
    } catch (TokenStreamException err) {
        // option
        throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine() + ", column " + lexer.getColumn() + ".  Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR);
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}