Examples with LowerCaseFilter org.apache.lucene.analysis.LowerCaseFilter used on opensource projects

Example 36 with LowerCaseFilter

use of org.apache.lucene.analysis.LowerCaseFilter in project lucene-solr by apache.

the class EnglishAnalyzer method normalize.

@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
    TokenStream result = new StandardFilter(in);
    result = new LowerCaseFilter(result);
    return result;
}

Example 37 with LowerCaseFilter

use of org.apache.lucene.analysis.LowerCaseFilter in project lucene-solr by apache.

the class JapaneseAnalyzer method normalize.

@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
    TokenStream result = new CJKWidthFilter(in);
    result = new LowerCaseFilter(result);
    return result;
}

Example 38 with LowerCaseFilter

use of org.apache.lucene.analysis.LowerCaseFilter in project lucene-solr by apache.

the class StandardAnalyzer method createComponents.

@Override
protected TokenStreamComponents createComponents(final String fieldName) {
    final StandardTokenizer src = new StandardTokenizer();
    src.setMaxTokenLength(maxTokenLength);
    TokenStream tok = new StandardFilter(src);
    tok = new LowerCaseFilter(tok);
    tok = new StopFilter(tok, stopwords);
    return new TokenStreamComponents(src, tok) {

        @Override
        protected void setReader(final Reader reader) {
            // So that if maxTokenLength was changed, the change takes
            // effect next time tokenStream is called:
            src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
            super.setReader(reader);
        }
    };
}

Example 39 with LowerCaseFilter

use of org.apache.lucene.analysis.LowerCaseFilter in project lucene-solr by apache.

the class StandardAnalyzer method normalize.

@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
    TokenStream result = new StandardFilter(in);
    result = new LowerCaseFilter(result);
    return result;
}

Example 40 with LowerCaseFilter

use of org.apache.lucene.analysis.LowerCaseFilter in project lucene-solr by apache.

the class PayloadSetter method testLowerCaseFilterLowSurrogateLeftover.

/**
   * Test that LowercaseFilter handles the lowercasing correctly if the term
   * buffer has a trailing surrogate character leftover and the current term in
   * the buffer ends with a corresponding leading surrogate.
   */
public void testLowerCaseFilterLowSurrogateLeftover() throws IOException {
    // test if the limit of the termbuffer is correctly used with supplementary
    // chars
    WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
    tokenizer.setReader(new StringReader("BogustermBogusterm�"));
    LowerCaseFilter filter = new LowerCaseFilter(tokenizer);
    assertTokenStreamContents(filter, new String[] { "bogustermbogusterm�" });
    filter.reset();
    String highSurEndingUpper = "BogustermBoguster�";
    String highSurEndingLower = "bogustermboguster�";
    tokenizer.setReader(new StringReader(highSurEndingUpper));
    assertTokenStreamContents(filter, new String[] { highSurEndingLower });
    assertTrue(filter.hasAttribute(CharTermAttribute.class));
    char[] termBuffer = filter.getAttribute(CharTermAttribute.class).buffer();
    int length = highSurEndingLower.length();
    assertEquals('�', termBuffer[length - 1]);
}