Examples with TokenMgrError net.sourceforge.pmd.lang.ast.TokenMgrError used on opensource projects

Example 1 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class ScalaTokenizer method tokenize.

public void tokenize(SourceCode source, Tokens cpdTokens) {
    String filename = source.getFileName();
    try {
        Lexer lexer = new Lexer();
        List<Token> tokens = lexer.getTokensOfFile(filename);
        for (Token token : tokens) {
            String tokenVal = token.tokenVal() != null ? token.tokenVal() : Integer.toString(token.tokenType());
            TokenEntry cpdToken = new TokenEntry(tokenVal, filename, token.line());
            cpdTokens.add(cpdToken);
        }
        cpdTokens.add(TokenEntry.getEOF());
    } catch (RuntimeException e) {
        e.printStackTrace();
        // option
        throw new TokenMgrError("Lexical error in file " + filename + ". The scala tokenizer exited with error: " + e.getMessage(), TokenMgrError.LEXICAL_ERROR);
    }
}

Example 2 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class ObjectiveCTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 3 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class CPPTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(maybeSkipBlocks(buffer.toString()));
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } catch (IOException e) {
        e.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 4 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class GroovyTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    GroovyLexer lexer = new GroovyLexer(IOUtils.toInputStream(buffer.toString()));
    TokenStream tokenStream = lexer.plumb();
    try {
        Token token = tokenStream.nextToken();
        while (token.getType() != Token.EOF_TYPE) {
            TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine());
            tokenEntries.add(tokenEntry);
            token = tokenStream.nextToken();
        }
    } catch (TokenStreamException err) {
        // option
        throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine() + ", column " + lexer.getColumn() + ".  Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR);
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}

Example 5 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class EcmascriptTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}