Examples with MockTokenizer org.apache.lucene.analysis.MockTokenizer used on opensource projects

Example 61 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestLimitTokenOffsetFilterFactory method test.

public void test() throws Exception {
    for (final boolean consumeAll : new boolean[] { true, false }) {
        Reader reader = new StringReader("A1 B2 C3 D4 E5 F6");
        MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
        tokenizer.setReader(reader);
        tokenizer.setEnableChecks(consumeAll);
        TokenStream stream = tokenizer;
        stream = tokenFilterFactory("LimitTokenOffset", LimitTokenOffsetFilterFactory.MAX_START_OFFSET, "3", LimitTokenOffsetFilterFactory.CONSUME_ALL_TOKENS_KEY, Boolean.toString(consumeAll)).create(stream);
        assertTokenStreamContents(stream, new String[] { "A1", "B2" });
    }
}

Example 62 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestFingerprintFilterFactory method test.

public void test() throws Exception {
    for (final boolean consumeAll : new boolean[] { true, false }) {
        Reader reader = new StringReader("A1 B2 A1 D4 C3");
        MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
        tokenizer.setReader(reader);
        tokenizer.setEnableChecks(consumeAll);
        TokenStream stream = tokenizer;
        stream = tokenFilterFactory("Fingerprint", FingerprintFilterFactory.MAX_OUTPUT_TOKEN_SIZE_KEY, "256", FingerprintFilterFactory.SEPARATOR_KEY, "_").create(stream);
        assertTokenStreamContents(stream, new String[] { "A1_B2_C3_D4" });
    }
}

Example 63 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestHyphenatedWordsFilter method testRandomString.

/** blast some random strings through the analyzer */
public void testRandomString() throws Exception {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
        }
    };
    checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
    a.close();
}

Example 64 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestHyphenatedWordsFilter method testOffsets.

public void testOffsets() throws Exception {
    String input = "abc- def geh 1234- 5678-";
    TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) ts).setReader(new StringReader(input));
    ts = new HyphenatedWordsFilter(ts);
    assertTokenStreamContents(ts, new String[] { "abcdef", "geh", "12345678-" }, new int[] { 0, 9, 13 }, new int[] { 8, 12, 24 });
}

Example 65 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestHyphenatedWordsFilter method testHyphenAtEnd.

/**
   * Test that HyphenatedWordsFilter behaves correctly with a final hyphen
   */
public void testHyphenAtEnd() throws Exception {
    String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-	hensive-hands-on and ecology-";
    // first test
    TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) ts).setReader(new StringReader(input));
    ts = new HyphenatedWordsFilter(ts);
    assertTokenStreamContents(ts, new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}