Examples with EdgeNGramTokenizer org.apache.lucene.analysis.ngram.EdgeNGramTokenizer used on opensource projects

Example 1 with EdgeNGramTokenizer

use of org.apache.lucene.analysis.ngram.EdgeNGramTokenizer in project lucene-solr by apache.

the class TestBugInSomething method testUnicodeShinglesAndNgrams.

// LUCENE-5269
@Slow
public void testUnicodeShinglesAndNgrams() throws Exception {
    Analyzer analyzer = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new EdgeNGramTokenizer(2, 94);
            //TokenStream stream = new SopTokenFilter(tokenizer);
            TokenStream stream = new ShingleFilter(tokenizer, 5);
            //stream = new SopTokenFilter(stream);
            stream = new NGramTokenFilter(stream, 55, 83);
            //stream = new SopTokenFilter(stream);
            return new TokenStreamComponents(tokenizer, stream);
        }
    };
    checkRandomData(random(), analyzer, 2000);
    analyzer.close();
}