Examples with MockTokenizer org.apache.lucene.analysis.MockTokenizer used on opensource projects

Example 21 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method testZeroSyns.

public void testZeroSyns() throws Exception {
    Tokenizer tokenizer = new MockTokenizer();
    tokenizer.setReader(new StringReader("aa bb"));
    try {
        new SynonymGraphFilter(tokenizer, new SynonymMap.Builder(true).build(), true);
        fail("did not hit expected exception");
    } catch (IllegalArgumentException iae) {
        // expected
        assertEquals("fst must be non-null", iae.getMessage());
    }
}

Example 22 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestApostropheFilterFactory method testApostrophes.

/**
   * Ensure the filter actually removes characters after an apostrophe.
   */
public void testApostrophes() throws Exception {
    Reader reader = new StringReader("Türkiye'de 2003'te Van Gölü'nü gördüm");
    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) stream).setReader(reader);
    stream = tokenFilterFactory("Apostrophe").create(stream);
    assertTokenStreamContents(stream, new String[] { "Türkiye", "2003", "Van", "Gölü", "gördüm" });
}

Example 23 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testRecursion.

public void testRecursion() throws Exception {
    b = new SynonymMap.Builder(true);
    final boolean keepOrig = false;
    add("zoo", "zoo", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
        }
    };
    assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "$", "zoo" }, new int[] { 1, 1, 1, 1 });
    a.close();
}

Example 24 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testDontKeepOrig.

public void testDontKeepOrig() throws Exception {
    b = new SynonymMap.Builder(true);
    add("a b", "foo", false);
    final SynonymMap map = b.build();
    final Analyzer analyzer = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
            return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
        }
    };
    assertAnalyzesTo(analyzer, "a b c", new String[] { "foo", "c" }, new int[] { 0, 4 }, new int[] { 3, 5 }, null, new int[] { 1, 1 }, new int[] { 1, 1 }, true);
    checkAnalysisConsistency(random(), analyzer, false, "a b c");
    analyzer.close();
}

Example 25 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testRepeatsOn.

public void testRepeatsOn() throws Exception {
    b = new SynonymMap.Builder(false);
    final boolean keepOrig = false;
    add("a b", "ab", keepOrig);
    add("a b", "ab", keepOrig);
    add("a b", "ab", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
        }
    };
    assertAnalyzesTo(a, "a b", new String[] { "ab", "ab", "ab" }, new int[] { 1, 0, 0 });
    a.close();
}