Examples with CharArraySet org.apache.lucene.analysis.CharArraySet used on opensource projects

Example 96 with CharArraySet

use of org.apache.lucene.analysis.CharArraySet in project lucene-solr by apache.

the class TestMorfologikAnalyzer method testKeywordAttrTokens.

/** */
public final void testKeywordAttrTokens() throws IOException {
    Analyzer a = new MorfologikAnalyzer() {

        @Override
        protected TokenStreamComponents createComponents(String field) {
            final CharArraySet keywords = new CharArraySet(1, false);
            keywords.add("liście");
            final Tokenizer src = new StandardTokenizer();
            TokenStream result = new StandardFilter(src);
            result = new SetKeywordMarkerFilter(result, keywords);
            result = new MorfologikFilter(result);
            return new TokenStreamComponents(src, result);
        }
    };
    assertAnalyzesTo(a, "liście danych", new String[] { "liście", "dany", "dana", "dane", "dać" }, new int[] { 0, 7, 7, 7, 7 }, new int[] { 6, 13, 13, 13, 13 }, new int[] { 1, 1, 0, 0, 0 });
    a.close();
}

Example 97 with CharArraySet

use of org.apache.lucene.analysis.CharArraySet in project lucene-solr by apache.

the class StopFilter method makeStopSet.

/**
   * Creates a stopword set from the given stopword list.
   * @param stopWords A List of Strings or char[] or any other toString()-able list representing the stopwords
   * @param ignoreCase if true, all words are lower cased first
   * @return A Set ({@link CharArraySet}) containing the words
   */
public static CharArraySet makeStopSet(List<?> stopWords, boolean ignoreCase) {
    CharArraySet stopSet = new CharArraySet(stopWords.size(), ignoreCase);
    stopSet.addAll(stopWords);
    return stopSet;
}

Example 98 with CharArraySet

use of org.apache.lucene.analysis.CharArraySet in project lucene-solr by apache.

the class StopFilter method makeStopSet.

/**
   * Creates a stopword set from the given stopword array.
   * 
   * @param stopWords An array of stopwords
   * @param ignoreCase If true, all words are lower cased first.  
   * @return a Set containing the words
   */
public static CharArraySet makeStopSet(String[] stopWords, boolean ignoreCase) {
    CharArraySet stopSet = new CharArraySet(stopWords.length, ignoreCase);
    stopSet.addAll(Arrays.asList(stopWords));
    return stopSet;
}

Example 99 with CharArraySet

use of org.apache.lucene.analysis.CharArraySet in project lucene-solr by apache.

the class TestSuggestStopFilter method testEndIsStopWord.

public void testEndIsStopWord() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to "));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go" }, new int[] { 0 }, new int[] { 2 }, null, new int[] { 1 }, null, 6, new boolean[] { false }, true);
}

Example 100 with CharArraySet

use of org.apache.lucene.analysis.CharArraySet in project lucene-solr by apache.

the class TestSuggestStopFilter method testMultipleStopWordsEnd.

public void testMultipleStopWordsEnd() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to", "the", "a");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to a the"));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go", "the" }, new int[] { 0, 8 }, new int[] { 2, 11 }, null, new int[] { 1, 3 }, null, 11, new boolean[] { false, true }, true);
}