Examples with JFlexSymbolMatcher org.opengrok.indexer.analysis.JFlexSymbolMatcher used on opensource projects

Example 1 with JFlexSymbolMatcher

use of org.opengrok.indexer.analysis.JFlexSymbolMatcher in project OpenGrok by OpenGrok.

the class PerlSymbolTokenizerTest method testOffsetAttribute.

/**
 * Helper method for {@link #testOffsetAttribute()} that runs the test on
 * one single implementation class with the specified input text and
 * expected tokens.
 */
private void testOffsetAttribute(Class<? extends JFlexSymbolMatcher> klass, String inputText, String[] expectedTokens) throws Exception {
    JFlexSymbolMatcher matcher = klass.getConstructor(Reader.class).newInstance(new StringReader(inputText));
    JFlexTokenizer tokenizer = new JFlexTokenizer(matcher);
    CharTermAttribute term = tokenizer.addAttribute(CharTermAttribute.class);
    OffsetAttribute offset = tokenizer.addAttribute(OffsetAttribute.class);
    int count = 0;
    while (tokenizer.incrementToken()) {
        assertTrue(count < expectedTokens.length, "too many tokens");
        String expected = expectedTokens[count];
        // 0-based offset to accord with String[]
        assertEquals(expected, term.toString(), "term" + count);
        assertEquals(inputText.indexOf(expected), offset.startOffset(), "start" + count);
        assertEquals(inputText.indexOf(expected) + expected.length(), offset.endOffset(), "end" + count);
        count++;
    }
    assertEquals(expectedTokens.length, count, "wrong number of tokens");
}