Search in sources :

Example 6 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class EcmascriptTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}
Also used : StringReader(java.io.StringReader) StringReader(java.io.StringReader) Reader(java.io.Reader) Token(net.sourceforge.pmd.lang.ecmascript5.ast.Token) TokenMgrError(net.sourceforge.pmd.lang.ast.TokenMgrError) LanguageVersionHandler(net.sourceforge.pmd.lang.LanguageVersionHandler) TokenManager(net.sourceforge.pmd.lang.TokenManager)

Example 7 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class PythonTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode);
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}
Also used : StringReader(java.io.StringReader) StringReader(java.io.StringReader) Reader(java.io.Reader) Token(net.sourceforge.pmd.lang.python.ast.Token) TokenMgrError(net.sourceforge.pmd.lang.ast.TokenMgrError) LanguageVersionHandler(net.sourceforge.pmd.lang.LanguageVersionHandler) TokenManager(net.sourceforge.pmd.lang.TokenManager)

Example 8 with TokenManager

use of net.sourceforge.pmd.lang.TokenManager in project pmd by pmd.

the class MatlabTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(MatlabLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}
Also used : StringReader(java.io.StringReader) StringReader(java.io.StringReader) Reader(java.io.Reader) Token(net.sourceforge.pmd.lang.matlab.ast.Token) TokenMgrError(net.sourceforge.pmd.lang.ast.TokenMgrError) LanguageVersionHandler(net.sourceforge.pmd.lang.LanguageVersionHandler) TokenManager(net.sourceforge.pmd.lang.TokenManager)

Aggregations

StringReader (java.io.StringReader)8 LanguageVersionHandler (net.sourceforge.pmd.lang.LanguageVersionHandler)8 TokenManager (net.sourceforge.pmd.lang.TokenManager)8 Reader (java.io.Reader)7 TokenMgrError (net.sourceforge.pmd.lang.ast.TokenMgrError)5 BufferedReader (java.io.BufferedReader)1 IOException (java.io.IOException)1 Token (net.sourceforge.pmd.lang.cpp.ast.Token)1 Token (net.sourceforge.pmd.lang.ecmascript5.ast.Token)1 Token (net.sourceforge.pmd.lang.java.ast.Token)1 Token (net.sourceforge.pmd.lang.jsp.ast.Token)1 Token (net.sourceforge.pmd.lang.matlab.ast.Token)1 Token (net.sourceforge.pmd.lang.objectivec.ast.Token)1 Token (net.sourceforge.pmd.lang.python.ast.Token)1 Token (net.sourceforge.pmd.lang.vf.ast.Token)1