Examples with MockTokenizer org.apache.lucene.analysis.MockTokenizer used on opensource projects

Example 41 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestGermanLightStemFilter method testKeyword.

public void testKeyword() throws IOException {
    final CharArraySet exclusionSet = new CharArraySet(asSet("sängerinnen"), false);
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
            return new TokenStreamComponents(source, new GermanLightStemFilter(sink));
        }
    };
    checkOneTerm(a, "sängerinnen", "sängerinnen");
    a.close();
}

Example 42 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestCzechStemFilterFactory method testStemming.

/**
   * Ensure the filter actually stems text.
   */
public void testStemming() throws Exception {
    Reader reader = new StringReader("angličtí");
    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) stream).setReader(reader);
    stream = tokenFilterFactory("CzechStem").create(stream);
    assertTokenStreamContents(stream, new String[] { "anglick" });
}

Example 43 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestCzechStemmer method testWithKeywordAttribute.

public void testWithKeywordAttribute() throws IOException {
    CharArraySet set = new CharArraySet(1, true);
    set.add("hole");
    final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    in.setReader(new StringReader("hole desek"));
    CzechStemFilter filter = new CzechStemFilter(new SetKeywordMarkerFilter(in, set));
    assertTokenStreamContents(filter, new String[] { "hole", "desk" });
}

Example 44 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestGermanNormalizationFilter method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    analyzer = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String field) {
            final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            final TokenStream stream = new GermanNormalizationFilter(tokenizer);
            return new TokenStreamComponents(tokenizer, stream);
        }
    };
}

Example 45 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSnowball method checkRandomStrings.

public void checkRandomStrings(final String snowballLanguage) throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer t = new MockTokenizer();
            return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
        }
    };
    checkRandomData(random(), a, 100 * RANDOM_MULTIPLIER);
    a.close();
}