Examples with Token org.sonar.plugins.scala.compiler.Token used on opensource projects

Example 1 with Token

use of org.sonar.plugins.scala.compiler.Token in project pmd by pmd.

the class ScalaTokenizer method tokenize.

public void tokenize(SourceCode source, Tokens cpdTokens) {
    String filename = source.getFileName();
    try {
        Lexer lexer = new Lexer();
        List<Token> tokens = lexer.getTokensOfFile(filename);
        for (Token token : tokens) {
            String tokenVal = token.tokenVal() != null ? token.tokenVal() : Integer.toString(token.tokenType());
            TokenEntry cpdToken = new TokenEntry(tokenVal, filename, token.line());
            cpdTokens.add(cpdToken);
        }
        cpdTokens.add(TokenEntry.getEOF());
    } catch (RuntimeException e) {
        e.printStackTrace();
        // option
        throw new TokenMgrError("Lexical error in file " + filename + ". The scala tokenizer exited with error: " + e.getMessage(), TokenMgrError.LEXICAL_ERROR);
    }
}