Examples with Token net.sourceforge.pmd.lang.cpp.ast.Token used on opensource projects

Example 1 with Token

use of net.sourceforge.pmd.lang.cpp.ast.Token in project pmd by pmd.

the class CPPTokenizerContinuationTest method parseWithContinuationCppTokenManager.

/**
 * Verifies the begin/end of a token. Uses the underlaying JavaCC Token and
 * not TokenEntry.
 */
@Test
public void parseWithContinuationCppTokenManager() throws Exception {
    String code = load("cpp_with_continuation.cpp");
    CppTokenManager tokenManager = new CppTokenManager(new StringReader(code));
    List<Token> tokens = new ArrayList<>();
    Token token = (Token) tokenManager.getNextToken();
    while (!token.image.isEmpty()) {
        tokens.add(token);
        token = (Token) tokenManager.getNextToken();
    }
    assertEquals(51, tokens.size());
    assertToken(tokens.get(2), "ab", 8, 12, 9, 1);
    assertToken(tokens.get(22), "\"2 Hello, world!\\n\"", 18, 16, 19, 9);
}

Example 2 with Token

use of net.sourceforge.pmd.lang.cpp.ast.Token in project pmd by pmd.

the class CPPTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(maybeSkipBlocks(buffer.toString()));
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } catch (IOException e) {
        e.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}