Examples with TokenManager net.sourceforge.pmd.lang.TokenManager used on opensource projects

Example 1 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class ObjectiveCTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 2 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class CPPTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(maybeSkipBlocks(buffer.toString()));
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } catch (IOException e) {
        e.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 3 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class VfTokenizer method tokenize.

public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
    Reader reader = null;
    try {
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenMgr.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenMgr.getNextToken();
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
    tokenEntries.add(TokenEntry.getEOF());
}

Example 4 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class JSPTokenizer method tokenize.

public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
    Reader reader = null;
    try {
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenMgr.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenMgr.getNextToken();
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
    tokenEntries.add(TokenEntry.getEOF());
}

Example 5 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class JavaTokenizer method tokenize.

public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder stringBuilder = sourceCode.getCodeBuffer();
    // Note that Java version is irrelevant for tokenizing
    LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME).getVersion("1.4").getLanguageVersionHandler();
    String fileName = sourceCode.getFileName();
    TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(fileName, new StringReader(stringBuilder.toString()));
    Token currentToken = (Token) tokenMgr.getNextToken();
    TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
    ConstructorDetector constructorDetector = new ConstructorDetector(ignoreIdentifiers);
    while (currentToken.image.length() > 0) {
        discarder.updateState(currentToken);
        if (discarder.isDiscarding()) {
            currentToken = (Token) tokenMgr.getNextToken();
            continue;
        }
        processToken(tokenEntries, fileName, currentToken, constructorDetector);
        currentToken = (Token) tokenMgr.getNextToken();
    }
    tokenEntries.add(TokenEntry.getEOF());
}