Examples with Token net.sourceforge.pmd.lang.vf.ast.Token used on opensource projects

Example 1 with Token

use of net.sourceforge.pmd.lang.vf.ast.Token in project pmd by pmd.

the class VfTokenizer method tokenize.

public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
    Reader reader = null;
    try {
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenMgr.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenMgr.getNextToken();
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
    tokenEntries.add(TokenEntry.getEOF());
}