Examples with Token net.sourceforge.pmd.lang.java.ast.Token used on opensource projects

Example 1 with Token

use of net.sourceforge.pmd.lang.java.ast.Token in project pmd by pmd.

the class JavaTokenizer method tokenize.

public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder stringBuilder = sourceCode.getCodeBuffer();
    // Note that Java version is irrelevant for tokenizing
    LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME).getVersion("1.4").getLanguageVersionHandler();
    String fileName = sourceCode.getFileName();
    TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(fileName, new StringReader(stringBuilder.toString()));
    Token currentToken = (Token) tokenMgr.getNextToken();
    TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
    ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
    while (currentToken.image.length() > 0) {
        discarder.updateState(currentToken);
        if (discarder.isDiscarding()) {
            currentToken = (Token) tokenMgr.getNextToken();
            continue;
        }
        processToken(tokenEntries, fileName, currentToken, constructorDetector);
        currentToken = (Token) tokenMgr.getNextToken();
    }
    tokenEntries.add(TokenEntry.getEOF());
}

Example 2 with Token

use of net.sourceforge.pmd.lang.java.ast.Token in project pmd by pmd.

the class AbstractCommentRuleTest method testFilteredCommentIn.

/**
 * Blank lines in comments should not raise an exception. See bug #1048.
 */
@Test
public void testFilteredCommentIn() {
    Token token = new Token();
    token.image = "/* multi line comment with blank lines\n\n\n */";
    String filtered = testSubject.filteredCommentIn(new MultiLineComment(token));
    assertEquals("multi line comment with blank lines", filtered);
    token.image = "/** a formal comment with blank lines\n\n\n */";
    filtered = testSubject.filteredCommentIn(new FormalComment(token));
    assertEquals("a formal comment with blank lines", filtered);
}