Examples with Tokenizer net.sourceforge.pmd.cpd.Tokenizer used on opensource projects

Example 1 with Tokenizer

use of net.sourceforge.pmd.cpd.Tokenizer in project sonarqube by SonarSource.

the class TokenizerBridgeTest method setUp.

@Before
public void setUp() {
    Tokenizer tokenizer = new Tokenizer() {

        public void tokenize(SourceCode tokens, Tokens tokenEntries) {
            tokenEntries.add(new TokenEntry("t1", "src", 1));
            tokenEntries.add(new TokenEntry("t2", "src", 1));
            tokenEntries.add(new TokenEntry("t3", "src", 2));
            tokenEntries.add(new TokenEntry("t1", "src", 4));
            tokenEntries.add(new TokenEntry("t3", "src", 4));
            tokenEntries.add(new TokenEntry("t3", "src", 4));
            tokenEntries.add(TokenEntry.getEOF());
        }
    };
    bridge = new TokenizerBridge(tokenizer, 10);
}