Examples with TokenMgrError net.sourceforge.pmd.lang.ast.TokenMgrError used on opensource projects

Example 6 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class ApexTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder code = sourceCode.getCodeBuffer();
    ANTLRStringStream ass = new ANTLRStringStream(code.toString());
    ApexLexer lexer = new ApexLexer(ass) {

        public void emitErrorMessage(String msg) {
            throw new TokenMgrError(msg, TokenMgrError.LEXICAL_ERROR);
        }
    };
    try {
        Token token = lexer.nextToken();
        while (token.getType() != Token.EOF) {
            if (token.getChannel() != Lexer.HIDDEN) {
                String tokenText = token.getText();
                if (!caseSensitive) {
                    tokenText = tokenText.toLowerCase(Locale.ROOT);
                }
                TokenEntry tokenEntry = new TokenEntry(tokenText, sourceCode.getFileName(), token.getLine());
                tokenEntries.add(tokenEntry);
            }
            token = lexer.nextToken();
        }
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}

Example 7 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class SwiftTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    try {
        ANTLRInputStream ais = new ANTLRInputStream(buffer.toString());
        SwiftLexer lexer = new SwiftLexer(ais);
        lexer.removeErrorListeners();
        lexer.addErrorListener(new ErrorHandler());
        Token token = lexer.nextToken();
        while (token.getType() != Token.EOF) {
            if (token.getChannel() != Lexer.HIDDEN) {
                TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine());
                tokenEntries.add(tokenEntry);
            }
            token = lexer.nextToken();
        }
    } catch (ANTLRSyntaxError err) {
        // option
        throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + err.getLine() + ", column " + err.getColumn() + ".  Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR);
    } finally {
        tokenEntries.add(TokenEntry.getEOF());
    }
}

Example 8 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class PythonTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode);
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 9 with TokenMgrError

use of net.sourceforge.pmd.lang.ast.TokenMgrError in project pmd by pmd.

the class MatlabTokenizer method tokenize.

@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder buffer = sourceCode.getCodeBuffer();
    Reader reader = null;
    try {
        LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(MatlabLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
        reader = new StringReader(buffer.toString());
        reader = IOUtil.skipBOM(reader);
        TokenManager tokenManager = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
        Token currentToken = (Token) tokenManager.getNextToken();
        while (currentToken.image.length() > 0) {
            tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
            currentToken = (Token) tokenManager.getNextToken();
        }
        tokenEntries.add(TokenEntry.getEOF());
        System.err.println("Added " + sourceCode.getFileName());
    } catch (TokenMgrError err) {
        err.printStackTrace();
        System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
        tokenEntries.add(TokenEntry.getEOF());
    } finally {
        IOUtils.closeQuietly(reader);
    }
}