Search in sources :

Example 11 with SnowballFilter

use of org.apache.lucene.analysis.snowball.SnowballFilter in project lucene-solr by apache.

the class DanishAnalyzer method createComponents.

/**
   * Creates a
   * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   * which tokenizes all the text in the provided {@link Reader}.
   * 
   * @return A
   *         {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   *         built from an {@link StandardTokenizer} filtered with
   *         {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}
   *         , {@link SetKeywordMarkerFilter} if a stem exclusion set is
   *         provided and {@link SnowballFilter}.
   */
@Override
protected TokenStreamComponents createComponents(String fieldName) {
    final Tokenizer source = new StandardTokenizer();
    TokenStream result = new StandardFilter(source);
    result = new LowerCaseFilter(result);
    result = new StopFilter(result, stopwords);
    if (!stemExclusionSet.isEmpty())
        result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new DanishStemmer());
    return new TokenStreamComponents(source, result);
}
Also used : TokenStream(org.apache.lucene.analysis.TokenStream) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) StopFilter(org.apache.lucene.analysis.StopFilter) SetKeywordMarkerFilter(org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter) DanishStemmer(org.tartarus.snowball.ext.DanishStemmer) StandardFilter(org.apache.lucene.analysis.standard.StandardFilter) SnowballFilter(org.apache.lucene.analysis.snowball.SnowballFilter) Tokenizer(org.apache.lucene.analysis.Tokenizer) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) LowerCaseFilter(org.apache.lucene.analysis.LowerCaseFilter)

Example 12 with SnowballFilter

use of org.apache.lucene.analysis.snowball.SnowballFilter in project lucene-solr by apache.

the class CatalanAnalyzer method createComponents.

/**
   * Creates a
   * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   * which tokenizes all the text in the provided {@link Reader}.
   * 
   * @return A
   *         {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   *         built from an {@link StandardTokenizer} filtered with
   *         {@link StandardFilter}, {@link ElisionFilter}, {@link LowerCaseFilter}, 
   *         {@link StopFilter}, {@link SetKeywordMarkerFilter} if a stem exclusion set is
   *         provided and {@link SnowballFilter}.
   */
@Override
protected TokenStreamComponents createComponents(String fieldName) {
    final Tokenizer source = new StandardTokenizer();
    TokenStream result = new StandardFilter(source);
    result = new ElisionFilter(result, DEFAULT_ARTICLES);
    result = new LowerCaseFilter(result);
    result = new StopFilter(result, stopwords);
    if (!stemExclusionSet.isEmpty())
        result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new CatalanStemmer());
    return new TokenStreamComponents(source, result);
}
Also used : TokenStream(org.apache.lucene.analysis.TokenStream) ElisionFilter(org.apache.lucene.analysis.util.ElisionFilter) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) StopFilter(org.apache.lucene.analysis.StopFilter) SetKeywordMarkerFilter(org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter) StandardFilter(org.apache.lucene.analysis.standard.StandardFilter) SnowballFilter(org.apache.lucene.analysis.snowball.SnowballFilter) Tokenizer(org.apache.lucene.analysis.Tokenizer) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) LowerCaseFilter(org.apache.lucene.analysis.LowerCaseFilter) CatalanStemmer(org.tartarus.snowball.ext.CatalanStemmer)

Example 13 with SnowballFilter

use of org.apache.lucene.analysis.snowball.SnowballFilter in project lucene-solr by apache.

the class TestLithuanianStemming method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, new LithuanianStemmer()));
        }
    };
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) SnowballFilter(org.apache.lucene.analysis.snowball.SnowballFilter) LithuanianStemmer(org.tartarus.snowball.ext.LithuanianStemmer) Analyzer(org.apache.lucene.analysis.Analyzer) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer)

Example 14 with SnowballFilter

use of org.apache.lucene.analysis.snowball.SnowballFilter in project lucene-solr by apache.

the class FinnishAnalyzer method createComponents.

/**
   * Creates a
   * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   * which tokenizes all the text in the provided {@link Reader}.
   * 
   * @return A
   *         {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   *         built from an {@link StandardTokenizer} filtered with
   *         {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}
   *         , {@link SetKeywordMarkerFilter} if a stem exclusion set is
   *         provided and {@link SnowballFilter}.
   */
@Override
protected TokenStreamComponents createComponents(String fieldName) {
    final Tokenizer source = new StandardTokenizer();
    TokenStream result = new StandardFilter(source);
    result = new LowerCaseFilter(result);
    result = new StopFilter(result, stopwords);
    if (!stemExclusionSet.isEmpty())
        result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    result = new SnowballFilter(result, new FinnishStemmer());
    return new TokenStreamComponents(source, result);
}
Also used : TokenStream(org.apache.lucene.analysis.TokenStream) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) StopFilter(org.apache.lucene.analysis.StopFilter) SetKeywordMarkerFilter(org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter) StandardFilter(org.apache.lucene.analysis.standard.StandardFilter) SnowballFilter(org.apache.lucene.analysis.snowball.SnowballFilter) FinnishStemmer(org.tartarus.snowball.ext.FinnishStemmer) Tokenizer(org.apache.lucene.analysis.Tokenizer) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) LowerCaseFilter(org.apache.lucene.analysis.LowerCaseFilter)

Example 15 with SnowballFilter

use of org.apache.lucene.analysis.snowball.SnowballFilter in project lucene-solr by apache.

the class TurkishAnalyzer method createComponents.

/**
   * Creates a
   * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   * which tokenizes all the text in the provided {@link Reader}.
   * 
   * @return A
   *         {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
   *         built from an {@link StandardTokenizer} filtered with
   *         {@link StandardFilter}, {@link TurkishLowerCaseFilter},
   *         {@link StopFilter}, {@link SetKeywordMarkerFilter} if a stem
   *         exclusion set is provided and {@link SnowballFilter}.
   */
@Override
protected TokenStreamComponents createComponents(String fieldName) {
    final Tokenizer source = new StandardTokenizer();
    TokenStream result = new StandardFilter(source);
    result = new ApostropheFilter(result);
    result = new TurkishLowerCaseFilter(result);
    result = new StopFilter(result, stopwords);
    if (!stemExclusionSet.isEmpty()) {
        result = new SetKeywordMarkerFilter(result, stemExclusionSet);
    }
    result = new SnowballFilter(result, new TurkishStemmer());
    return new TokenStreamComponents(source, result);
}
Also used : TokenStream(org.apache.lucene.analysis.TokenStream) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer) StopFilter(org.apache.lucene.analysis.StopFilter) SetKeywordMarkerFilter(org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter) TurkishStemmer(org.tartarus.snowball.ext.TurkishStemmer) StandardFilter(org.apache.lucene.analysis.standard.StandardFilter) SnowballFilter(org.apache.lucene.analysis.snowball.SnowballFilter) Tokenizer(org.apache.lucene.analysis.Tokenizer) StandardTokenizer(org.apache.lucene.analysis.standard.StandardTokenizer)

Aggregations

SnowballFilter (org.apache.lucene.analysis.snowball.SnowballFilter)18 TokenStream (org.apache.lucene.analysis.TokenStream)17 Tokenizer (org.apache.lucene.analysis.Tokenizer)16 StopFilter (org.apache.lucene.analysis.StopFilter)15 StandardTokenizer (org.apache.lucene.analysis.standard.StandardTokenizer)15 SetKeywordMarkerFilter (org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter)14 StandardFilter (org.apache.lucene.analysis.standard.StandardFilter)14 LowerCaseFilter (org.apache.lucene.analysis.LowerCaseFilter)13 ElisionFilter (org.apache.lucene.analysis.util.ElisionFilter)2 LithuanianStemmer (org.tartarus.snowball.ext.LithuanianStemmer)2 Analyzer (org.apache.lucene.analysis.Analyzer)1 MockTokenizer (org.apache.lucene.analysis.MockTokenizer)1 EnglishPossessiveFilter (org.apache.lucene.analysis.en.EnglishPossessiveFilter)1 StemmerOverrideFilter (org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter)1 TurkishLowerCaseFilter (org.apache.lucene.analysis.tr.TurkishLowerCaseFilter)1 ArmenianStemmer (org.tartarus.snowball.ext.ArmenianStemmer)1 BasqueStemmer (org.tartarus.snowball.ext.BasqueStemmer)1 CatalanStemmer (org.tartarus.snowball.ext.CatalanStemmer)1 DanishStemmer (org.tartarus.snowball.ext.DanishStemmer)1 FinnishStemmer (org.tartarus.snowball.ext.FinnishStemmer)1