Examples with TokenStream org.apache.lucene.analysis.TokenStream used on opensource projects

Example 86 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestPortugueseLightStemFilterFactory method testStemming.

public void testStemming() throws Exception {
    Reader reader = new StringReader("evidentemente");
    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) stream).setReader(reader);
    stream = tokenFilterFactory("PortugueseLightStem").create(stream);
    assertTokenStreamContents(stream, new String[] { "evident" });
}

Example 87 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestPortugueseLightStemFilter method testKeyword.

public void testKeyword() throws IOException {
    final CharArraySet exclusionSet = new CharArraySet(asSet("quilométricas"), false);
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
            return new TokenStreamComponents(source, new PortugueseLightStemFilter(sink));
        }
    };
    checkOneTerm(a, "quilométricas", "quilométricas");
    a.close();
}

Example 88 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestPortugueseMinimalStemFilter method testKeyword.

public void testKeyword() throws IOException {
    final CharArraySet exclusionSet = new CharArraySet(asSet("quilométricas"), false);
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
            return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(sink));
        }
    };
    checkOneTerm(a, "quilométricas", "quilométricas");
    a.close();
}

Example 89 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestPortugueseMinimalStemFilterFactory method testStemming.

public void testStemming() throws Exception {
    Reader reader = new StringReader("questões");
    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) stream).setReader(reader);
    stream = tokenFilterFactory("PortugueseMinimalStem").create(stream);
    assertTokenStreamContents(stream, new String[] { "questão" });
}

Example 90 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestPortugueseStemFilter method testKeyword.

public void testKeyword() throws IOException {
    final CharArraySet exclusionSet = new CharArraySet(asSet("quilométricas"), false);
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
            return new TokenStreamComponents(source, new PortugueseStemFilter(sink));
        }
    };
    checkOneTerm(a, "quilométricas", "quilométricas");
    a.close();
}